path: root/tools/lint/test/test_perfdocs_generation.py

import os
import pathlib
from unittest import mock

import mozunit

LINTER = "perfdocs"


def setup_sample_logger(logger, structured_logger, top_dir):
    from perfdocs.logger import PerfDocLogger

    PerfDocLogger.LOGGER = structured_logger
    PerfDocLogger.PATHS = ["perfdocs"]
    PerfDocLogger.TOP_DIR = top_dir

    import perfdocs.gatherer as gt
    import perfdocs.generator as gn
    import perfdocs.utils as utls
    import perfdocs.verifier as vf

    gt.logger = logger
    vf.logger = logger
    gn.logger = logger
    utls.logger = logger


@mock.patch("perfdocs.logger.PerfDocLogger")
def test_perfdocs_generator_generate_perfdocs_pass(
    logger, structured_logger, perfdocs_sample
):
    from test_perfdocs import temp_file

    top_dir = perfdocs_sample["top_dir"]
    setup_sample_logger(logger, structured_logger, top_dir)

    templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
    templates_dir.mkdir(parents=True, exist_ok=True)

    from perfdocs.generator import Generator
    from perfdocs.verifier import Verifier

    verifier = Verifier(top_dir)
    verifier.validate_tree()

    generator = Generator(verifier, generate=True, workspace=top_dir)
    with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
        generator.generate_perfdocs()

    assert logger.warning.call_count == 0


@mock.patch("perfdocs.logger.PerfDocLogger")
def test_perfdocs_generator_generate_perfdocs_metrics_pass(
    logger, structured_logger, perfdocs_sample
):
    from test_perfdocs import temp_file

    top_dir = perfdocs_sample["top_dir"]
    setup_sample_logger(logger, structured_logger, top_dir)

    templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
    templates_dir.mkdir(parents=True, exist_ok=True)

    from perfdocs.generator import Generator
    from perfdocs.verifier import Verifier

    sample_gatherer_result = {
        "suite": {"Example": {"metrics": ["fcp", "SpeedIndex"]}},
        "another_suite": {"Example": {}},
    }
    sample_test_list_result = {
        "suite": [{"metrics": ["fcp", "SpeedIndex"], "name": "Example"}],
        "another_suite": [{"name": "Example"}],
    }

    with temp_file(
        "metrics.rst",
        tempdir=pathlib.Path(top_dir, "perfdocs"),
        content="{metrics_documentation}",
    ):
        with mock.patch(
            "perfdocs.framework_gatherers.RaptorGatherer.get_test_list"
        ) as m:
            m.return_value = sample_gatherer_result
            with perfdocs_sample["config"].open("w") as f1, perfdocs_sample[
                "config_metrics"
            ].open("r") as f2:
                # Overwrite content of config.yml with metrics config
                f1.write(f2.read())

            verifier = Verifier(top_dir)
            verifier.validate_tree()

            verifier._gatherer.framework_gatherers[
                "raptor"
            ]._descriptions = sample_test_list_result

        generator = Generator(verifier, generate=True, workspace=top_dir)
        with temp_file(
            "index.rst", tempdir=templates_dir, content="{test_documentation}"
        ):
            generator.generate_perfdocs()

        with pathlib.Path(generator.perfdocs_path, "raptor-metrics.rst").open() as f:
            metrics_content = f.read()
            assert "{metrics_documentation}" not in metrics_content
            assert "a description" in metrics_content
            assert "**Tests using it**" in metrics_content

        with pathlib.Path(generator.perfdocs_path, "raptor.rst").open() as f:
            raptor_index_content = f.read()
            assert "{metrics_rst_name}" not in raptor_index_content
            assert "raptor-metrics" in raptor_index_content

    assert logger.warning.call_count == 0


@mock.patch("perfdocs.logger.PerfDocLogger")
def test_perfdocs_generator_needed_regeneration(
    logger, structured_logger, perfdocs_sample
):
    top_dir = perfdocs_sample["top_dir"]
    setup_sample_logger(logger, structured_logger, top_dir)

    from perfdocs.generator import Generator
    from perfdocs.verifier import Verifier

    verifier = Verifier(top_dir)
    verifier.validate_tree()

    generator = Generator(verifier, generate=False, workspace=top_dir)
    generator.generate_perfdocs()

    expected = "PerfDocs need to be regenerated."
    args, _ = logger.warning.call_args

    assert logger.warning.call_count == 1
    assert args[0] == expected


@mock.patch("perfdocs.generator.get_changed_files", new=lambda x: [])
@mock.patch("perfdocs.generator.ON_TRY", new=True)
@mock.patch("perfdocs.logger.PerfDocLogger")
def test_perfdocs_generator_needed_update(logger, structured_logger, perfdocs_sample):
    from test_perfdocs import temp_file

    top_dir = perfdocs_sample["top_dir"]
    setup_sample_logger(logger, structured_logger, top_dir)

    templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
    templates_dir.mkdir(parents=True, exist_ok=True)

    from perfdocs.generator import Generator
    from perfdocs.verifier import Verifier

    # Initializing perfdocs
    verifier = Verifier(top_dir)
    verifier.validate_tree()

    generator = Generator(verifier, generate=True, workspace=top_dir)
    with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
        generator.generate_perfdocs()

        # Removed file for testing and run again
        generator._generate = False
        files = [f for f in os.listdir(generator.perfdocs_path)]
        for f in files:
            os.remove(str(pathlib.Path(generator.perfdocs_path, f)))

        generator.generate_perfdocs()

    expected = (
        "PerfDocs are outdated, run ./mach lint -l perfdocs --fix .` to update them. "
        "You can also apply the perfdocs.diff patch file produced from this "
        "reviewbot test to fix the issue."
    )
    args, _ = logger.warning.call_args

    assert logger.warning.call_count == 1
    assert args[0] == expected

    # Check to ensure a diff was produced
    assert logger.log.call_count == 6

    logs = [v[0][0] for v in logger.log.call_args_list]
    for failure_log in (
        "Some files are missing or are funny.",
        "Missing in existing docs: index.rst",
        "Missing in existing docs: mozperftest.rst",
    ):
        assert failure_log in logs


@mock.patch("perfdocs.generator.get_changed_files", new=lambda x: [])
@mock.patch("perfdocs.generator.ON_TRY", new=True)
def test_perfdocs_generator_update_with_no_changes(structured_logger, perfdocs_sample):
    """This test ensures that when no changed files exist, we'll still trigger a failure."""
    from perfdocs.logger import PerfDocLogger
    from test_perfdocs import temp_file

    top_dir = perfdocs_sample["top_dir"]

    logger_mock = mock.MagicMock()
    PerfDocLogger.LOGGER = logger_mock
    PerfDocLogger.PATHS = ["perfdocs"]
    PerfDocLogger.TOP_DIR = top_dir
    logger = PerfDocLogger()

    setup_sample_logger(logger, logger_mock, top_dir)

    templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
    templates_dir.mkdir(parents=True, exist_ok=True)

    from perfdocs.generator import Generator
    from perfdocs.verifier import Verifier

    # Initializing perfdocs
    verifier = Verifier(top_dir)
    verifier.validate_tree()

    generator = Generator(verifier, generate=True, workspace=top_dir)
    with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
        generator.generate_perfdocs()

        # Removed file for testing and run again
        generator._generate = False
        files = [f for f in os.listdir(generator.perfdocs_path)]
        for f in files:
            os.remove(str(pathlib.Path(generator.perfdocs_path, f)))

        generator.generate_perfdocs()

    expected = (
        "PerfDocs are outdated, run ./mach lint -l perfdocs --fix .` to update them. "
        "You can also apply the perfdocs.diff patch file produced from this "
        "reviewbot test to fix the issue."
    )
    assert logger.LOGGER.lint_error.call_args is not None
    _, msg = logger.LOGGER.lint_error.call_args

    assert logger.FAILED
    assert logger.LOGGER.lint_error.call_count == 1
    assert msg["message"] == expected
    assert msg["rule"] == "Flawless performance docs (unknown file)"


@mock.patch("perfdocs.logger.PerfDocLogger")
def test_perfdocs_generator_created_perfdocs(
    logger, structured_logger, perfdocs_sample
):
    from test_perfdocs import temp_file

    top_dir = perfdocs_sample["top_dir"]
    setup_sample_logger(logger, structured_logger, top_dir)

    templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
    templates_dir.mkdir(parents=True, exist_ok=True)

    from perfdocs.generator import Generator
    from perfdocs.verifier import Verifier

    verifier = Verifier(top_dir)
    verifier.validate_tree()

    generator = Generator(verifier, generate=True, workspace=top_dir)
    with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
        perfdocs_tmpdir = generator._create_perfdocs()

    files = [f for f in os.listdir(perfdocs_tmpdir)]
    files.sort()
    expected_files = ["index.rst", "mozperftest.rst"]

    for i, file in enumerate(files):
        assert file == expected_files[i]

    with pathlib.Path(perfdocs_tmpdir, expected_files[0]).open() as f:
        filedata = f.readlines()
    assert "".join(filedata) == "  * :doc:`mozperftest`"


@mock.patch("perfdocs.logger.PerfDocLogger")
def test_perfdocs_generator_build_perfdocs(logger, structured_logger, perfdocs_sample):
    top_dir = perfdocs_sample["top_dir"]
    setup_sample_logger(logger, structured_logger, top_dir)

    from perfdocs.generator import Generator
    from perfdocs.verifier import Verifier

    verifier = Verifier(top_dir)
    verifier.validate_tree()

    generator = Generator(verifier, generate=True, workspace=top_dir)
    frameworks_info = generator.build_perfdocs_from_tree()

    expected = ["dynamic", "metrics", "static"]

    for framework in sorted(frameworks_info.keys()):
        for i, framework_info in enumerate(frameworks_info[framework]):
            assert framework_info == expected[i]


@mock.patch("perfdocs.logger.PerfDocLogger")
def test_perfdocs_generator_create_temp_dir(logger, structured_logger, perfdocs_sample):
    top_dir = perfdocs_sample["top_dir"]
    setup_sample_logger(logger, structured_logger, top_dir)

    from perfdocs.generator import Generator
    from perfdocs.verifier import Verifier

    verifier = Verifier(top_dir)
    verifier.validate_tree()

    generator = Generator(verifier, generate=True, workspace=top_dir)
    tmpdir = generator._create_temp_dir()

    assert pathlib.Path(tmpdir).is_dir()


@mock.patch("perfdocs.logger.PerfDocLogger")
def test_perfdocs_generator_create_temp_dir_fail(
    logger, structured_logger, perfdocs_sample
):
    top_dir = perfdocs_sample["top_dir"]
    setup_sample_logger(logger, structured_logger, top_dir)

    from perfdocs.generator import Generator
    from perfdocs.verifier import Verifier

    verifier = Verifier(top_dir)
    verifier.validate_tree()

    generator = Generator(verifier, generate=True, workspace=top_dir)
    with mock.patch("perfdocs.generator.pathlib") as path_mock:
        path_mock.Path().mkdir.side_effect = OSError()
        path_mock.Path().is_dir.return_value = False
        tmpdir = generator._create_temp_dir()

    expected = "Error creating temp file: "
    args, _ = logger.critical.call_args

    assert not tmpdir
    assert logger.critical.call_count == 1
    assert args[0] == expected


@mock.patch("perfdocs.logger.PerfDocLogger")
def test_perfdocs_generator_save_perfdocs_pass(
    logger, structured_logger, perfdocs_sample
):
    from test_perfdocs import temp_file

    top_dir = perfdocs_sample["top_dir"]
    setup_sample_logger(logger, structured_logger, top_dir)

    templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
    templates_dir.mkdir(parents=True, exist_ok=True)

    from perfdocs.generator import Generator
    from perfdocs.verifier import Verifier

    verifier = Verifier(top_dir)
    verifier.validate_tree()

    generator = Generator(verifier, generate=True, workspace=top_dir)

    assert not generator.perfdocs_path.is_dir()

    with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
        perfdocs_tmpdir = generator._create_perfdocs()

    generator._save_perfdocs(perfdocs_tmpdir)

    expected = ["index.rst", "mozperftest.rst"]
    files = [f for f in os.listdir(generator.perfdocs_path)]
    files.sort()

    for i, file in enumerate(files):
        assert file == expected[i]


@mock.patch("perfdocs.generator.shutil")
@mock.patch("perfdocs.logger.PerfDocLogger")
def test_perfdocs_generator_save_perfdocs_fail(
    logger, shutil, structured_logger, perfdocs_sample
):
    from test_perfdocs import temp_file

    top_dir = perfdocs_sample["top_dir"]
    setup_sample_logger(logger, structured_logger, top_dir)

    templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
    templates_dir.mkdir(parents=True, exist_ok=True)

    from perfdocs.generator import Generator
    from perfdocs.verifier import Verifier

    verifier = Verifier(top_dir)
    verifier.validate_tree()

    generator = Generator(verifier, generate=True, workspace=top_dir)
    with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
        perfdocs_tmpdir = generator._create_perfdocs()

    shutil.copytree = mock.Mock(side_effect=Exception())
    generator._save_perfdocs(perfdocs_tmpdir)

    expected = "There was an error while saving the documentation: "
    args, _ = logger.critical.call_args

    assert logger.critical.call_count == 1
    assert args[0] == expected


if __name__ == "__main__":
    mozunit.main()