3 files changed, 103 insertions, 0 deletions
diff --git a/tools/lint/test-manifest-alpha.yml b/tools/lint/test-manifest-alpha.yml
new file mode 100644
index 0000000000..cd4761e4b6
--- /dev/null
+++ b/tools/lint/test-manifest-alpha.yml
@@ -0,0 +1,19 @@
+---
+test-manifest-alpha:
+    description: Mochitest manifest tests should be in alphabetical order.
+    exclude:
+        - "**/application.ini"
+        - "**/l10n.ini"
+        - "**/xpcshell.ini"
+        - "**/python.ini"
+        - "**/manifest.ini"
+        - dom/canvas/test/webgl-conf/mochitest-errata.ini
+        - python/mozbuild/mozbuild/test/backend/data
+        - testing/mozbase/manifestparser/tests
+        - testing/web-platform
+        - xpcom/tests/unit/data
+    extensions: ['ini']
+    type: external
+    payload: test-manifest-alpha:lint
+    support-files:
+        - 'tools/lint/test-manifest-alpha/error-level-manifests.yml'
diff --git a/tools/lint/test-manifest-alpha/__init__.py b/tools/lint/test-manifest-alpha/__init__.py
new file mode 100644
index 0000000000..87d6ce5c5d
--- /dev/null
+++ b/tools/lint/test-manifest-alpha/__init__.py
@@ -0,0 +1,77 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import difflib
+import os
+import sys
+
+import yaml
+from manifestparser import TestManifest
+from mozlint import result
+from mozlint.pathutils import expand_exclusions
+from mozpack import path as mozpath
+
+# Since this linter matches on all files with .ini extensions, we can omit
+# those extensions from this allowlist, which makes it easier to match
+# against variants like "mochitest-serviceworker.ini".
+FILENAME_ALLOWLIST = ["mochitest", "browser", "chrome", "a11y"]
+
+here = os.path.abspath(os.path.dirname(__file__))
+ERROR_LEVEL_MANIFESTS_PATH = os.path.join(here, "error-level-manifests.yml")
+
+
+def lint(paths, config, fix=None, **lintargs):
+    try:
+        with open(ERROR_LEVEL_MANIFESTS_PATH) as f:
+            error_level_manifests = yaml.safe_load(f)
+    except (IOError, ValueError) as e:
+        print("{}: error:\n  {}".format(ERROR_LEVEL_MANIFESTS_PATH, e), file=sys.stderr)
+        sys.exit(1)
+
+    topsrcdir = lintargs["root"]
+
+    results = []
+    file_names = list(expand_exclusions(paths, config, lintargs["root"]))
+
+    for file_name in file_names:
+        name = os.path.basename(file_name)
+        if not any(name.startswith(allowed) for allowed in FILENAME_ALLOWLIST):
+            continue
+
+        manifest = TestManifest(manifests=(file_name,), strict=False)
+
+        test_names = [test["name"] for test in manifest.tests]
+        sorted_test_names = sorted(test_names)
+
+        if test_names != sorted_test_names:
+            rel_file_path = mozpath.relpath(file_name, topsrcdir)
+            level = "warning"
+
+            if (mozpath.normsep(rel_file_path) in error_level_manifests) or (
+                any(
+                    mozpath.match(rel_file_path, e)
+                    for e in error_level_manifests
+                    if "*" in e
+                )
+            ):
+                level = "error"
+
+            diff_instance = difflib.Differ()
+            diff_result = diff_instance.compare(test_names, sorted_test_names)
+            diff_list = list(diff_result)
+
+            res = {
+                "path": rel_file_path,
+                "lineno": 0,
+                "column": 0,
+                "message": (
+                    "The mochitest test manifest is not in alphabetical order. "
+                    "Expected ordering: \n\n%s\n\n" % "\n".join(sorted_test_names)
+                ),
+                "level": level,
+                "diff": "\n".join(diff_list),
+            }
+            results.append(result.from_config(config, **res))
+
+    return {"results": results, "fixed": 0}
diff --git a/tools/lint/test-manifest-alpha/error-level-manifests.yml b/tools/lint/test-manifest-alpha/error-level-manifests.yml
new file mode 100644
index 0000000000..90c091a3b8
--- /dev/null
+++ b/tools/lint/test-manifest-alpha/error-level-manifests.yml
@@ -0,0 +1,7 @@
+---
+# This file contains a list of manifest files or directory patterns that have
+# have been put into alphabetical order already. Items in this list will
+# cause the test-manifest-alpha linter to use the Error level rather than
+# the Warning level.
+
+- browser/**