From 1fd6a618b60d7168fd8f37585d5d39d22d775afd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 28 Mar 2024 07:11:39 +0100 Subject: Adding upstream version 0.13.0. Signed-off-by: Daniel Baumann --- docs/README.md | 70 +++++ docs/advanced_usages/as-python-lib.md | 315 ++++++++++++++++++++ docs/advanced_usages/caching.md | 87 ++++++ docs/advanced_usages/custom-tests.md | 324 +++++++++++++++++++++ docs/api/catalog.md | 13 + docs/api/device.md | 25 ++ docs/api/inventory.md | 11 + docs/api/inventory.models.input.md | 13 + docs/api/models.md | 37 +++ docs/api/report_manager.md | 7 + docs/api/result_manager.md | 15 + docs/api/result_manager_models.md | 15 + docs/api/tests.aaa.md | 13 + docs/api/tests.bfd.md | 13 + docs/api/tests.configuration.md | 13 + docs/api/tests.connectivity.md | 13 + docs/api/tests.field_notices.md | 13 + docs/api/tests.hardware.md | 13 + docs/api/tests.interfaces.md | 13 + docs/api/tests.logging.md | 13 + docs/api/tests.md | 37 +++ docs/api/tests.mlag.md | 13 + docs/api/tests.multicast.md | 13 + docs/api/tests.profiles.md | 13 + docs/api/tests.routing.bgp.md | 13 + docs/api/tests.routing.generic.md | 13 + docs/api/tests.routing.ospf.md | 13 + docs/api/tests.security.md | 13 + docs/api/tests.services.md | 13 + docs/api/tests.snmp.md | 13 + docs/api/tests.software.md | 13 + docs/api/tests.stp.md | 13 + docs/api/tests.system.md | 13 + docs/api/tests.vlan.md | 13 + docs/api/tests.vxlan.md | 13 + docs/api/types.md | 10 + docs/cli/check.md | 36 +++ docs/cli/debug.md | 175 +++++++++++ docs/cli/exec.md | 298 +++++++++++++++++++ docs/cli/get-inventory-information.md | 237 +++++++++++++++ docs/cli/inv-from-ansible.md | 72 +++++ docs/cli/inv-from-cvp.md | 72 +++++ docs/cli/nrfu.md | 247 ++++++++++++++++ docs/cli/overview.md | 103 +++++++ docs/cli/tag-management.md | 165 +++++++++++ docs/contribution.md | 227 +++++++++++++++ docs/faq.md | 67 +++++ docs/getting-started.md | 301 +++++++++++++++++++ docs/imgs/animated-svg.md | 8 + docs/imgs/anta-nrfu-json-output.png | Bin 0 -> 48313 bytes docs/imgs/anta-nrfu-table-filter-host-output.png | Bin 0 -> 54158 bytes docs/imgs/anta-nrfu-table-filter-test-output.png | Bin 0 -> 37593 bytes docs/imgs/anta-nrfu-table-group-by-host-output.png | Bin 0 -> 37720 bytes docs/imgs/anta-nrfu-table-group-by-test-output.png | Bin 0 -> 40229 bytes docs/imgs/anta-nrfu-table-output.png | Bin 0 -> 273402 bytes docs/imgs/anta-nrfu-table-per-host-output.png | Bin 0 -> 21836 bytes docs/imgs/anta-nrfu-text-output.png | Bin 0 -> 43588 bytes docs/imgs/anta-nrfu-tpl-report-output.png | Bin 0 -> 22567 bytes docs/imgs/anta-nrfu.cast | 64 ++++ docs/imgs/anta-nrfu.svg | 1 + docs/imgs/favicon.ico | Bin 0 -> 4286 bytes docs/imgs/uml/anta.device.AntaDevice.jpeg | Bin 0 -> 5125 bytes docs/imgs/uml/anta.device.AsyncEOSDevice.jpeg | Bin 0 -> 31385 bytes docs/imgs/uml/anta.models.AntaCommand.jpeg | Bin 0 -> 21486 bytes docs/imgs/uml/anta.models.AntaTemplate.jpeg | Bin 0 -> 7613 bytes docs/imgs/uml/anta.models.AntaTest.jpeg | Bin 0 -> 27186 bytes .../uml/anta.result_manager.ResultManager.jpeg | Bin 0 -> 10960 bytes .../uml/anta.result_manager.models.TestResult.jpeg | Bin 0 -> 8875 bytes docs/overrides/main.html | 17 ++ docs/requirements-and-installation.md | 105 +++++++ docs/scripts/generate_svg.py | 92 ++++++ docs/snippets/anta_help.txt | 20 ++ docs/stylesheets/extra.material.css | 207 +++++++++++++ docs/stylesheets/highlight.js | 3 + docs/stylesheets/tables.js | 6 + docs/usage-inventory-catalog.md | 246 ++++++++++++++++ 76 files changed, 4034 insertions(+) create mode 100755 docs/README.md create mode 100644 docs/advanced_usages/as-python-lib.md create mode 100644 docs/advanced_usages/caching.md create mode 100644 docs/advanced_usages/custom-tests.md create mode 100644 docs/api/catalog.md create mode 100644 docs/api/device.md create mode 100644 docs/api/inventory.md create mode 100644 docs/api/inventory.models.input.md create mode 100644 docs/api/models.md create mode 100644 docs/api/report_manager.md create mode 100644 docs/api/result_manager.md create mode 100644 docs/api/result_manager_models.md create mode 100644 docs/api/tests.aaa.md create mode 100644 docs/api/tests.bfd.md create mode 100644 docs/api/tests.configuration.md create mode 100644 docs/api/tests.connectivity.md create mode 100644 docs/api/tests.field_notices.md create mode 100644 docs/api/tests.hardware.md create mode 100644 docs/api/tests.interfaces.md create mode 100644 docs/api/tests.logging.md create mode 100644 docs/api/tests.md create mode 100644 docs/api/tests.mlag.md create mode 100644 docs/api/tests.multicast.md create mode 100644 docs/api/tests.profiles.md create mode 100644 docs/api/tests.routing.bgp.md create mode 100644 docs/api/tests.routing.generic.md create mode 100644 docs/api/tests.routing.ospf.md create mode 100644 docs/api/tests.security.md create mode 100644 docs/api/tests.services.md create mode 100644 docs/api/tests.snmp.md create mode 100644 docs/api/tests.software.md create mode 100644 docs/api/tests.stp.md create mode 100644 docs/api/tests.system.md create mode 100644 docs/api/tests.vlan.md create mode 100644 docs/api/tests.vxlan.md create mode 100644 docs/api/types.md create mode 100644 docs/cli/check.md create mode 100644 docs/cli/debug.md create mode 100644 docs/cli/exec.md create mode 100644 docs/cli/get-inventory-information.md create mode 100644 docs/cli/inv-from-ansible.md create mode 100644 docs/cli/inv-from-cvp.md create mode 100644 docs/cli/nrfu.md create mode 100644 docs/cli/overview.md create mode 100644 docs/cli/tag-management.md create mode 100644 docs/contribution.md create mode 100644 docs/faq.md create mode 100644 docs/getting-started.md create mode 100644 docs/imgs/animated-svg.md create mode 100755 docs/imgs/anta-nrfu-json-output.png create mode 100644 docs/imgs/anta-nrfu-table-filter-host-output.png create mode 100644 docs/imgs/anta-nrfu-table-filter-test-output.png create mode 100644 docs/imgs/anta-nrfu-table-group-by-host-output.png create mode 100644 docs/imgs/anta-nrfu-table-group-by-test-output.png create mode 100644 docs/imgs/anta-nrfu-table-output.png create mode 100755 docs/imgs/anta-nrfu-table-per-host-output.png create mode 100755 docs/imgs/anta-nrfu-text-output.png create mode 100755 docs/imgs/anta-nrfu-tpl-report-output.png create mode 100644 docs/imgs/anta-nrfu.cast create mode 100644 docs/imgs/anta-nrfu.svg create mode 100644 docs/imgs/favicon.ico create mode 100644 docs/imgs/uml/anta.device.AntaDevice.jpeg create mode 100644 docs/imgs/uml/anta.device.AsyncEOSDevice.jpeg create mode 100644 docs/imgs/uml/anta.models.AntaCommand.jpeg create mode 100644 docs/imgs/uml/anta.models.AntaTemplate.jpeg create mode 100644 docs/imgs/uml/anta.models.AntaTest.jpeg create mode 100644 docs/imgs/uml/anta.result_manager.ResultManager.jpeg create mode 100644 docs/imgs/uml/anta.result_manager.models.TestResult.jpeg create mode 100644 docs/overrides/main.html create mode 100644 docs/requirements-and-installation.md create mode 100644 docs/scripts/generate_svg.py create mode 100644 docs/snippets/anta_help.txt create mode 100644 docs/stylesheets/extra.material.css create mode 100644 docs/stylesheets/highlight.js create mode 100644 docs/stylesheets/tables.js create mode 100644 docs/usage-inventory-catalog.md (limited to 'docs') diff --git a/docs/README.md b/docs/README.md new file mode 100755 index 0000000..d47d3fe --- /dev/null +++ b/docs/README.md @@ -0,0 +1,70 @@ + + +[![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](https://github.com/arista-netdevops-community/anta/blob/main/LICENSE) +[![Linting and Testing Anta](https://github.com/arista-netdevops-community/anta/actions/workflows/code-testing.yml/badge.svg)](https://github.com/arista-netdevops-community/anta/actions/workflows/code-testing.yml) +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +![GitHub commit activity (branch)](https://img.shields.io/github/commit-activity/m/arista-netdevops-community/anta) +[![github release](https://img.shields.io/github/release/arista-netdevops-community/anta.svg)](https://github.com/arista-netdevops-community/anta/releases/) +![PyPI - Downloads](https://img.shields.io/pypi/dm/anta) +![coverage](https://raw.githubusercontent.com/arista-netdevops-community/anta/coverage-badge/latest-release-coverage.svg) + +# Arista Network Test Automation (ANTA) Framework + +ANTA is Python framework that automates tests for Arista devices. + +- ANTA provides a [set of tests](api/tests.md) to validate the state of your network +- ANTA can be used to: + - Automate NRFU (Network Ready For Use) test on a preproduction network + - Automate tests on a live network (periodically or on demand) +- ANTA can be used with: + - The [ANTA CLI](cli/overview.md) + - As a [Python library](advanced_usages/as-python-lib.md) in your own application + +![anta nrfu](https://raw.githubusercontent.com/arista-netdevops-community/anta/main/docs/imgs/anta-nrfu.svg) + +```bash +# Install ANTA CLI +$ pip install anta + +# Run ANTA CLI +$ anta --help +Usage: anta [OPTIONS] COMMAND [ARGS]... + + Arista Network Test Automation (ANTA) CLI + +Options: + --version Show the version and exit. + --log-file FILE Send the logs to a file. If logging level is + DEBUG, only INFO or higher will be sent to + stdout. [env var: ANTA_LOG_FILE] + -l, --log-level [CRITICAL|ERROR|WARNING|INFO|DEBUG] + ANTA logging level [env var: + ANTA_LOG_LEVEL; default: INFO] + --help Show this message and exit. + +Commands: + check Commands to validate configuration files + debug Commands to execute EOS commands on remote devices + exec Commands to execute various scripts on EOS devices + get Commands to get information from or generate inventories + nrfu Run ANTA tests on devices +``` + +> [!WARNING] +> The ANTA CLI options have changed after version 0.11 and have moved away from the top level `anta` and are now required at their respective commands (e.g. `anta nrfu`). This breaking change occurs after users feedback on making the CLI more intuitive. This change should not affect user experience when using environment variables. + +## Documentation + +The documentation is published on [ANTA package website](https://www.anta.ninja). Also, a [demo repository](https://github.com/titom73/atd-anta-demo) is available to facilitate your journey with ANTA. + +## Contribution guide + +Contributions are welcome. Please refer to the [contribution guide](contribution.md) + +## Credits + +Thank you to [Angélique Phillipps](https://github.com/aphillipps), [Colin MacGiollaEáin](https://github.com/colinmacgiolla), [Khelil Sator](https://github.com/ksator), [Matthieu Tache](https://github.com/mtache), [Onur Gashi](https://github.com/onurgashi), [Paul Lavelle](https://github.com/paullavelle), [Guillaume Mulocher](https://github.com/gmuloc) and [Thomas Grimonet](https://github.com/titom73) for their contributions and guidances. diff --git a/docs/advanced_usages/as-python-lib.md b/docs/advanced_usages/as-python-lib.md new file mode 100644 index 0000000..b4ce654 --- /dev/null +++ b/docs/advanced_usages/as-python-lib.md @@ -0,0 +1,315 @@ + + +ANTA is a Python library that can be used in user applications. This section describes how you can leverage ANTA Python modules to help you create your own NRFU solution. + +!!! tip + If you are unfamiliar with asyncio, refer to the Python documentation relevant to your Python version - https://docs.python.org/3/library/asyncio.html + +## [AntaDevice](../api/device.md#anta.device.AntaDevice) Abstract Class + +A device is represented in ANTA as a instance of a subclass of the [AntaDevice](../api/device.md### ::: anta.device.AntaDevice) abstract class. +There are few abstract methods that needs to be implemented by child classes: + +- The [collect()](../api/device.md#anta.device.AntaDevice.collect) coroutine is in charge of collecting outputs of [AntaCommand](../api/models.md#anta.models.AntaCommand) instances. +- The [refresh()](../api/device.md#anta.device.AntaDevice.refresh) coroutine is in charge of updating attributes of the [AntaDevice](../api/device.md### ::: anta.device.AntaDevice) instance. These attributes are used by [AntaInventory](../api/inventory.md#anta.inventory.AntaInventory) to filter out unreachable devices or by [AntaTest](../api/models.md#anta.models.AntaTest) to skip devices based on their hardware models. + +The [copy()](../api/device.md#anta.device.AntaDevice.copy) coroutine is used to copy files to and from the device. It does not need to be implemented if tests are not using it. + +### [AsyncEOSDevice](../api/device.md#anta.device.AsyncEOSDevice) Class + +The [AsyncEOSDevice](../api/device.md#anta.device.AsyncEOSDevice) class is an implementation of [AntaDevice](../api/device.md#anta.device.AntaDevice) for Arista EOS. +It uses the [aio-eapi](https://github.com/jeremyschulman/aio-eapi) eAPI client and the [AsyncSSH](https://github.com/ronf/asyncssh) library. + +- The [collect()](../api/device.md#anta.device.AsyncEOSDevice.collect) coroutine collects [AntaCommand](../api/models.md#anta.models.AntaCommand) outputs using eAPI. +- The [refresh()](../api/device.md#anta.device.AsyncEOSDevice.refresh) coroutine tries to open a TCP connection on the eAPI port and update the `is_online` attribute accordingly. If the TCP connection succeeds, it sends a `show version` command to gather the hardware model of the device and updates the `established` and `hw_model` attributes. +- The [copy()](../api/device.md#anta.device.AsyncEOSDevice.copy) coroutine copies files to and from the device using the SCP protocol. + +## [AntaInventory](../api/inventory.md#anta.inventory.AntaInventory) Class + +The [AntaInventory](../api/inventory.md#anta.inventory.AntaInventory) class is a subclass of the standard Python type [dict](https://docs.python.org/3/library/stdtypes.html#dict). The keys of this dictionary are the device names, the values are [AntaDevice](../api/device.md#anta.device.AntaDevice) instances. + + +[AntaInventory](../api/inventory.md#anta.inventory.AntaInventory) provides methods to interact with the ANTA inventory: + +- The [add_device()](../api/inventory.md#anta.inventory.AntaInventory.add_device) method adds an [AntaDevice](../api/device.md### ::: anta.device.AntaDevice) instance to the inventory. Adding an entry to [AntaInventory](../api/inventory.md#anta.inventory.AntaInventory) with a key different from the device name is not allowed. +- The [get_inventory()](../api/inventory.md#anta.inventory.AntaInventory.get_inventory) returns a new [AntaInventory](../api/inventory.md#anta.inventory.AntaInventory) instance with filtered out devices based on the method inputs. +- The [connect_inventory()](../api/inventory.md#anta.inventory.AntaInventory.connect_inventory) coroutine will execute the [refresh()](../api/device.md#anta.device.AntaDevice.refresh) coroutines of all the devices in the inventory. +- The [parse()](../api/inventory.md#anta.inventory.AntaInventory.parse) static method creates an [AntaInventory](../api/inventory.md#anta.inventory.AntaInventory) instance from a YAML file and returns it. The devices are [AsyncEOSDevice](../api/device.md#anta.device.AsyncEOSDevice) instances. + + +To parse a YAML inventory file and print the devices connection status: + +```python +""" +Example +""" +import asyncio + +from anta.inventory import AntaInventory + + +async def main(inv: AntaInventory) -> None: + """ + Take an AntaInventory and: + 1. try to connect to every device in the inventory + 2. print a message for every device connection status + """ + await inv.connect_inventory() + + for device in inv.values(): + if device.established: + print(f"Device {device.name} is online") + else: + print(f"Could not connect to device {device.name}") + +if __name__ == "__main__": + # Create the AntaInventory instance + inventory = AntaInventory.parse( + filename="inv.yml", + username="arista", + password="@rista123", + timeout=15, + ) + + # Run the main coroutine + res = asyncio.run(main(inventory)) +``` + +??? note "How to create your inventory file" + Please visit this [dedicated section](../usage-inventory-catalog.md) for how to use inventory and catalog files. + +To run an EOS commands list on the reachable devices from the inventory: +```python +""" +Example +""" +# This is needed to run the script for python < 3.10 for typing annotations +from __future__ import annotations + +import asyncio +from pprint import pprint + +from anta.inventory import AntaInventory +from anta.models import AntaCommand + + +async def main(inv: AntaInventory, commands: list[str]) -> dict[str, list[AntaCommand]]: + """ + Take an AntaInventory and a list of commands as string and: + 1. try to connect to every device in the inventory + 2. collect the results of the commands from each device + + Returns: + a dictionary where key is the device name and the value is the list of AntaCommand ran towards the device + """ + await inv.connect_inventory() + + # Make a list of coroutine to run commands towards each connected device + coros = [] + # dict to keep track of the commands per device + result_dict = {} + for name, device in inv.get_inventory(established_only=True).items(): + anta_commands = [AntaCommand(command=command, ofmt="json") for command in commands] + result_dict[name] = anta_commands + coros.append(device.collect_commands(anta_commands)) + + # Run the coroutines + await asyncio.gather(*coros) + + return result_dict + + +if __name__ == "__main__": + # Create the AntaInventory instance + inventory = AntaInventory.parse( + filename="inv.yml", + username="arista", + password="@rista123", + timeout=15, + ) + + # Create a list of commands with json output + commands = ["show version", "show ip bgp summary"] + + # Run the main asyncio entry point + res = asyncio.run(main(inventory, commands)) + + pprint(res) +``` + + +## Use tests from ANTA + +All the test classes inherit from the same abstract Base Class AntaTest. The Class definition indicates which commands are required for the test and the user should focus only on writing the `test` function with optional keywords argument. The instance of the class upon creation instantiates a TestResult object that can be accessed later on to check the status of the test ([unset, skipped, success, failure, error]). + +### Test structure + +All tests are built on a class named `AntaTest` which provides a complete toolset for a test: + +- Object creation +- Test definition +- TestResult definition +- Abstracted method to collect data + +This approach means each time you create a test it will be based on this `AntaTest` class. Besides that, you will have to provide some elements: + +- `name`: Name of the test +- `description`: A human readable description of your test +- `categories`: a list of categories to sort test. +- `commands`: a list of command to run. This list _must_ be a list of `AntaCommand` which is described in the next part of this document. + +Here is an example of a hardware test related to device temperature: + +```python +from __future__ import annotations + +import logging +from typing import Any, Dict, List, Optional, cast + +from anta.models import AntaTest, AntaCommand + + +class VerifyTemperature(AntaTest): + """ + Verifies device temparture is currently OK. + """ + + # The test name + name = "VerifyTemperature" + # A small description of the test, usually the first line of the class docstring + description = "Verifies device temparture is currently OK" + # The category of the test, usually the module name + categories = ["hardware"] + # The command(s) used for the test. Could be a template instead + commands = [AntaCommand(command="show system environment temperature", ofmt="json")] + + # Decorator + @AntaTest.anta_test + # abstract method that must be defined by the child Test class + def test(self) -> None: + """Run VerifyTemperature validation""" + command_output = cast(Dict[str, Dict[Any, Any]], self.instance_commands[0].output) + temperature_status = command_output["systemStatus"] if "systemStatus" in command_output.keys() else "" + if temperature_status == "temperatureOk": + self.result.is_success() + else: + self.result.is_failure(f"Device temperature is not OK, systemStatus: {temperature_status }") +``` + +When you run the test, object will automatically call its `anta.models.AntaTest.collect()` method to get device output for each command if no pre-collected data was given to the test. This method does a loop to call `anta.inventory.models.InventoryDevice.collect()` methods which is in charge of managing device connection and how to get data. + +??? info "run test offline" + You can also pass eos data directly to your test if you want to validate data collected in a different workflow. An example is provided below just for information: + + ```python + test = VerifyTemperature(device, eos_data=test_data["eos_data"]) + asyncio.run(test.test()) + ``` + +The `test` function is always the same and __must__ be defined with the `@AntaTest.anta_test` decorator. This function takes at least one argument which is a `anta.inventory.models.InventoryDevice` object. +In some cases a test would rely on some additional inputs from the user, for instance the number of expected peers or some expected numbers. All parameters __must__ come with a default value and the test function __should__ validate the parameters values (at this stage this is the only place where validation can be done but there are future plans to make this better). + +```python +class VerifyTemperature(AntaTest): + ... + @AntaTest.anta_test + def test(self) -> None: + pass + +class VerifyTransceiversManufacturers(AntaTest): + ... + @AntaTest.anta_test + def test(self, manufacturers: Optional[List[str]] = None) -> None: + # validate the manufactures parameter + pass +``` + +The test itself does not return any value, but the result is directly availble from your AntaTest object and exposes a `anta.result_manager.models.TestResult` object with result, name of the test and optional messages: + + +- `name` (str): Device name where the test has run. +- `test` (str): Test name runs on the device. +- `categories` (List[str]): List of categories the TestResult belongs to, by default the AntaTest categories. +- `description` (str): TestResult description, by default the AntaTest description. +- `results` (str): Result of the test. Can be one of ["unset", "success", "failure", "error", "skipped"]. +- `message` (str, optional): Message to report after the test if any. +- `custom_field` (str, optional): Custom field to store a string for flexibility in integrating with ANTA + +```python +from anta.tests.hardware import VerifyTemperature + +test = VerifyTemperature(device, eos_data=test_data["eos_data"]) +asyncio.run(test.test()) +assert test.result.result == "success" +``` + +### Classes for commands + +To make it easier to get data, ANTA defines 2 different classes to manage commands to send to devices: + +#### [AntaCommand](../api/models.md#anta.models.AntaCommand) Class + +Represent a command with following information: + +- Command to run +- Ouput format expected +- eAPI version +- Output of the command + +Usage example: + +```python +from anta.models import AntaCommand + +cmd1 = AntaCommand(command="show zerotouch") +cmd2 = AntaCommand(command="show running-config diffs", ofmt="text") +``` + +!!! tip "Command revision and version" + * Most of EOS commands return a JSON structure according to a model (some commands may not be modeled hence the necessity to use `text` outformat sometimes. + * The model can change across time (adding feature, ... ) and when the model is changed in a non backward-compatible way, the __revision__ number is bumped. The initial model starts with __revision__ 1. + * A __revision__ applies to a particular CLI command whereas a __version__ is global to an eAPI call. The __version__ is internally translated to a specific __revision__ for each CLI command in the RPC call. The currently supported __version__ vaues are `1` and `latest`. + * A __revision takes precedence over a version__ (e.g. if a command is run with version="latest" and revision=1, the first revision of the model is returned) + * By default eAPI returns the first revision of each model to ensure that when upgrading, intergation with existing tools is not broken. This is done by using by default `version=1` in eAPI calls. + + ANTA uses by default `version="latest"` in AntaCommand. For some commands, you may want to run them with a different revision or version. + + For instance the `VerifyRoutingTableSize` test leverages the first revision of `show bfd peers`: + + ``` + # revision 1 as later revision introduce additional nesting for type + commands = [AntaCommand(command="show bfd peers", revision=1)] + ``` + +#### [AntaTemplate](../api/models.md#anta.models.AntaTemplate) Class + +Because some command can require more dynamic than just a command with no parameter provided by user, ANTA supports command template: you define a template in your test class and user provide parameters when creating test object. + +```python + +class RunArbitraryTemplateCommand(AntaTest): + """ + Run an EOS command and return result + Based on AntaTest to build relevant output for pytest + """ + + name = "Run aributrary EOS command" + description = "To be used only with anta debug commands" + template = AntaTemplate(template="show interfaces {ifd}") + categories = ["debug"] + + @AntaTest.anta_test + def test(self) -> None: + errdisabled_interfaces = [interface for interface, value in response["interfaceStatuses"].items() if value["linkStatus"] == "errdisabled"] + ... + + +params = [{"ifd": "Ethernet2"}, {"ifd": "Ethernet49/1"}] +run_command1 = RunArbitraryTemplateCommand(device_anta, params) +``` + +In this example, test waits for interfaces to check from user setup and will only check for interfaces in `params` diff --git a/docs/advanced_usages/caching.md b/docs/advanced_usages/caching.md new file mode 100644 index 0000000..cec2467 --- /dev/null +++ b/docs/advanced_usages/caching.md @@ -0,0 +1,87 @@ + + +ANTA is a streamlined Python framework designed for efficient interaction with network devices. This section outlines how ANTA incorporates caching mechanisms to collect command outputs from network devices. + +## Configuration + +By default, ANTA utilizes [aiocache](https://github.com/aio-libs/aiocache)'s memory cache backend, also called [`SimpleMemoryCache`](https://aiocache.aio-libs.org/en/v0.12.2/caches.html#simplememorycache). This library aims for simplicity and supports asynchronous operations to go along with Python `asyncio` used in ANTA. + +The `_init_cache()` method of the [AntaDevice](../advanced_usages/as-python-lib.md#antadevice-abstract-class) abstract class initializes the cache. Child classes can override this method to tweak the cache configuration: + +```python +def _init_cache(self) -> None: + """ + Initialize cache for the device, can be overridden by subclasses to manipulate how it works + """ + self.cache = Cache(cache_class=Cache.MEMORY, ttl=60, namespace=self.name, plugins=[HitMissRatioPlugin()]) + self.cache_locks = defaultdict(asyncio.Lock) +``` + +The cache is also configured with `aiocache`'s [`HitMissRatioPlugin`](https://aiocache.aio-libs.org/en/v0.12.2/plugins.html#hitmissratioplugin) plugin to calculate the ratio of hits the cache has and give useful statistics for logging purposes in ANTA. + +## Cache key design + +The cache is initialized per `AntaDevice` and uses the following cache key design: + +`:` + +The `uid` is an attribute of [AntaCommand](../advanced_usages/as-python-lib.md#antacommand-class), which is a unique identifier generated from the command, version, revision and output format. + +Each UID has its own asyncio lock. This design allows coroutines that need to access the cache for different UIDs to do so concurrently. The locks are managed by the `self.cache_locks` dictionary. + +## Mechanisms + +By default, once the cache is initialized, it is used in the `collect()` method of `AntaDevice`. The `collect()` method prioritizes retrieving the output of the command from the cache. If the output is not in the cache, the private `_collect()` method will retrieve and then store it for future access. + +## How to disable caching + +Caching is enabled by default in ANTA following the previous configuration and mechanisms. + +There might be scenarios where caching is not wanted. You can disable caching in multiple ways in ANTA: + +1. Caching can be disabled globally, for **ALL** commands on **ALL** devices, using the `--disable-cache` global flag when invoking anta at the [CLI](../cli/overview.md#invoking-anta-cli): + ```bash + anta --disable-cache --username arista --password arista nrfu table + ``` +2. Caching can be disabled per device, network or range by setting the `disable_cache` key to `True` when definining the ANTA [Inventory](../usage-inventory-catalog.md#create-an-inventory-file) file: + ```yaml + anta_inventory: + hosts: + - host: 172.20.20.101 + name: DC1-SPINE1 + tags: ["SPINE", "DC1"] + disable_cache: True # Set this key to True + - host: 172.20.20.102 + name: DC1-SPINE2 + tags: ["SPINE", "DC1"] + disable_cache: False # Optional since it's the default + + networks: + - network: "172.21.21.0/24" + disable_cache: True + + ranges: + - start: 172.22.22.10 + end: 172.22.22.19 + disable_cache: True + ``` + This approach effectively disables caching for **ALL** commands sent to devices targeted by the `disable_cache` key. + +3. For tests developpers, caching can be disabled for a specific [`AntaCommand`](../advanced_usages/as-python-lib.md#antacommand-class) or [`AntaTemplate`](../advanced_usages/as-python-lib.md#antatemplate-class) by setting the `use_cache` attribute to `False`. That means the command output will always be collected on the device and therefore, never use caching. + +### Disable caching in a child class of `AntaDevice` + +Since caching is implemented at the `AntaDevice` abstract class level, all subclasses will inherit that default behavior. As a result, if you need to disable caching in any custom implementation of `AntaDevice` outside of the ANTA framework, you must initialize `AntaDevice` with `disable_cache` set to `True`: + +```python +class AnsibleEOSDevice(AntaDevice): + """ + Implementation of an AntaDevice using Ansible HttpApi plugin for EOS. + """ + def __init__(self, name: str, connection: ConnectionBase, tags: list = None) -> None: + super().__init__(name, tags, disable_cache=True) +``` diff --git a/docs/advanced_usages/custom-tests.md b/docs/advanced_usages/custom-tests.md new file mode 100644 index 0000000..87402c1 --- /dev/null +++ b/docs/advanced_usages/custom-tests.md @@ -0,0 +1,324 @@ + + +!!! info "" + This documentation applies for both creating tests in ANTA or creating your own test package. + +ANTA is not only a Python library with a CLI and a collection of built-in tests, it is also a framework you can extend by building your own tests. + +## Generic approach + +A test is a Python class where a test function is defined and will be run by the framework. + +ANTA provides an abstract class [AntaTest](../api/models.md#anta.models.AntaTest). This class does the heavy lifting and provide the logic to define, collect and test data. The code below is an example of a simple test in ANTA, which is an [AntaTest](../api/models.md#anta.models.AntaTest) subclass: + +```python +from anta.models import AntaTest, AntaCommand +from anta.decorators import skip_on_platforms + + +class VerifyTemperature(AntaTest): + """ + This test verifies if the device temperature is within acceptable limits. + + Expected Results: + * success: The test will pass if the device temperature is currently OK: 'temperatureOk'. + * failure: The test will fail if the device temperature is NOT OK. + """ + + name = "VerifyTemperature" + description = "Verifies if the device temperature is within the acceptable range." + categories = ["hardware"] + commands = [AntaCommand(command="show system environment temperature", ofmt="json")] + + @skip_on_platforms(["cEOSLab", "vEOS-lab"]) + @AntaTest.anta_test + def test(self) -> None: + command_output = self.instance_commands[0].json_output + temperature_status = command_output["systemStatus"] if "systemStatus" in command_output.keys() else "" + if temperature_status == "temperatureOk": + self.result.is_success() + else: + self.result.is_failure(f"Device temperature exceeds acceptable limits. Current system status: '{temperature_status}'") +``` + +[AntaTest](../api/models.md#anta.models.AntaTest) also provide more advanced capabilities like [AntaCommand](../api/models.md#anta.models.AntaCommand) templating using the [AntaTemplate](../api/models.md#anta.models.AntaTemplate) class or test inputs definition and validation using [AntaTest.Input](../api/models.md#anta.models.AntaTest.Input) [pydantic](https://docs.pydantic.dev/latest/) model. This will be discussed in the sections below. + +## [AntaTest](../api/models.md#anta.models.AntaTest) structure + +### Class Attributes + +- `name` (`str`): Name of the test. Used during reporting. +- `description` (`str`): A human readable description of your test. +- `categories` (`list[str]`): A list of categories in which the test belongs. +- `commands` (`list[Union[AntaTemplate, AntaCommand]]`): A list of command to collect from devices. This list __must__ be a list of [AntaCommand](../api/models.md#anta.models.AntaCommand) or [AntaTemplate](../api/models.md#anta.models.AntaTemplate) instances. Rendering [AntaTemplate](../api/models.md#anta.models.AntaTemplate) instances will be discussed later. + +!!! info + All these class attributes are mandatory. If any attribute is missing, a `NotImplementedError` exception will be raised during class instantiation. + +### Instance Attributes + +!!! info + You can access an instance attribute in your code using the `self` reference. E.g. you can access the test input values using `self.inputs`. + +::: anta.models.AntaTest + options: + show_docstring_attributes: true + show_root_heading: false + show_bases: false + show_docstring_description: false + show_docstring_examples: false + show_docstring_parameters: false + members: false + show_source: false + show_root_toc_entry: false + heading_level: 10 + + +!!! note "Logger object" + ANTA already provides comprehensive logging at every steps of a test execution. The [AntaTest](../api/models.md#anta.models.AntaTest) class also provides a `logger` attribute that is a Python logger specific to the test instance. See [Python documentation](https://docs.python.org/3/library/logging.html) for more information. + +!!! note "AntaDevice object" + Even if `device` is not a private attribute, you should not need to access this object in your code. + +### Test Inputs + +[AntaTest.Input](../api/models.md#anta.models.AntaTest.Input) is a [pydantic model](https://docs.pydantic.dev/latest/usage/models/) that allow test developers to define their test inputs. [pydantic](https://docs.pydantic.dev/latest/) provides out of the box [error handling](https://docs.pydantic.dev/latest/usage/models/#error-handling) for test input validation based on the type hints defined by the test developer. + +The base definition of [AntaTest.Input](../api/models.md#anta.models.AntaTest.Input) provides common test inputs for all [AntaTest](../api/models.md#anta.models.AntaTest) instances: + +#### [Input](../api/models.md#anta.models.AntaTest.Input) model + +::: anta.models.AntaTest.Input + options: + show_docstring_attributes: true + show_root_heading: false + show_category_heading: false + show_bases: false + show_docstring_description: false + show_docstring_examples: false + show_docstring_parameters: false + show_source: false + members: false + show_root_toc_entry: false + heading_level: 10 + +#### [ResultOverwrite](../api/models.md#anta.models.AntaTest.Input.ResultOverwrite) model + +::: anta.models.AntaTest.Input.ResultOverwrite + options: + show_docstring_attributes: true + show_root_heading: false + show_category_heading: false + show_bases: false + show_docstring_description: false + show_docstring_examples: false + show_docstring_parameters: false + show_source: false + show_root_toc_entry: false + heading_level: 10 + +!!! note + The pydantic model is configured using the [`extra=forbid`](https://docs.pydantic.dev/latest/usage/model_config/#extra-attributes) that will fail input validation if extra fields are provided. + +### Methods + +- [test(self) -> None](../api/models.md#anta.models.AntaTest.test): This is an abstract method that __must__ be implemented. It contains the test logic that can access the collected command outputs using the `instance_commands` instance attribute, access the test inputs using the `inputs` instance attribute and __must__ set the `result` instance attribute accordingly. It must be implemented using the `AntaTest.anta_test` decorator that provides logging and will collect commands before executing the `test()` method. +- [render(self, template: AntaTemplate) -> list[AntaCommand]](../api/models.md#anta.models.AntaTest.render): This method only needs to be implemented if [AntaTemplate](../api/models.md#anta.models.AntaTemplate) instances are present in the `commands` class attribute. It will be called for every [AntaTemplate](../api/models.md#anta.models.AntaTemplate) occurence and __must__ return a list of [AntaCommand](../api/models.md#anta.models.AntaCommand) using the [AntaTemplate.render()](../api/models.md#anta.models.AntaTemplate.render) method. It can access test inputs using the `inputs` instance attribute. + +## Test execution + +Below is a high level description of the test execution flow in ANTA: + +1. ANTA will parse the test catalog to get the list of [AntaTest](../api/models.md#anta.models.AntaTest) subclasses to instantiate and their associated input values. We consider a single [AntaTest](../api/models.md#anta.models.AntaTest) subclass in the following steps. + +2. ANTA will instantiate the [AntaTest](../api/models.md#anta.models.AntaTest) subclass and a single device will be provided to the test instance. The `Input` model defined in the class will also be instantiated at this moment. If any [ValidationError](https://docs.pydantic.dev/latest/errors/errors/) is raised, the test execution will be stopped. + +3. If there is any [AntaTemplate](../api/models.md#anta.models.AntaTemplate) instance in the `commands` class attribute, [render()](../api/models.md#anta.models.AntaTest.render) will be called for every occurrence. At this moment, the `instance_commands` attribute has been initialized. If any rendering error occurs, the test execution will be stopped. + +4. The `AntaTest.anta_test` decorator will collect the commands from the device and update the `instance_commands` attribute with the outputs. If any collection error occurs, the test execution will be stopped. + +5. The [test()](../api/models.md#anta.models.AntaTest.test) method is executed. + +## Writing an AntaTest subclass + +In this section, we will go into all the details of writing an [AntaTest](../api/models.md#anta.models.AntaTest) subclass. + +### Class definition + +Import [anta.models.AntaTest](../api/models.md#anta.models.AntaTest) and define your own class. +Define the mandatory class attributes using [anta.models.AntaCommand](../api/models.md#anta.models.AntaCommand), [anta.models.AntaTemplate](../api/models.md#anta.models.AntaTemplate) or both. + +!!! info + Caching can be disabled per `AntaCommand` or `AntaTemplate` by setting the `use_cache` argument to `False`. For more details about how caching is implemented in ANTA, please refer to [Caching in ANTA](../advanced_usages/caching.md). + +```python +from anta.models import AntaTest, AntaCommand, AntaTemplate + + +class (AntaTest): + """ + + """ + + name = "YourTestName" # should be your class name + description = "" + categories = ["", ""] + commands = [ + AntaCommand( + command="", + ofmt="", + version="", + revision="", # revision has precedence over version + use_cache="", + ), + AntaTemplate( + template="", + ofmt="", + version="", + revision="", # revision has precedence over version + use_cache="", + ) + ] +``` + +### Inputs definition + +If the user needs to provide inputs for your test, you need to define a [pydantic model](https://docs.pydantic.dev/latest/usage/models/) that defines the schema of the test inputs: + +```python +class (AntaTest): + ... + class Input(AntaTest.Input): # pylint: disable=missing-class-docstring + : + """""" +``` + +To define an input field type, refer to the [pydantic documentation](https://docs.pydantic.dev/latest/usage/types/types/) about types. +You can also leverage [anta.custom_types](../api/types.md) that provides reusable types defined in ANTA tests. + +Regarding required, optional and nullable fields, refer to this [documentation](https://docs.pydantic.dev/latest/migration/#required-optional-and-nullable-fields) on how to define them. + +!!! note + All the `pydantic` features are supported. For instance you can define [validators](https://docs.pydantic.dev/latest/usage/validators/) for complex input validation. + +### Template rendering + +Define the `render()` method if you have [AntaTemplate](../api/models.md#anta.models.AntaTemplate) instances in your `commands` class attribute: + +```python +class (AntaTest): + ... + def render(self, template: AntaTemplate) -> list[AntaCommand]: + return [template.render(