Merging upstream version 10.1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

5 files changed, 158 insertions, 4 deletions

diff --git a/python/clippy/__init__.py b/python/clippy/__init__.py
index 60119fb..668724a 100644
--- a/python/clippy/__init__.py
+++ b/python/clippy/__init__.py
@@ -20,6 +20,7 @@ from _clippy import (
     CMD_ATTR_HIDDEN,
     CMD_ATTR_DEPRECATED,
     CMD_ATTR_NOSH,
+    elf_notes,
 )
 
 
diff --git a/python/clippy/elf.py b/python/clippy/elf.py
index cc442ee..fd34842 100644
--- a/python/clippy/elf.py
+++ b/python/clippy/elf.py
@@ -458,7 +458,16 @@ class ELFSubset(object):
         - `this[123:str]` - extract until null byte.  The slice stop value is
             the `str` type (or, technically, `unicode`.)
         """
-        return self._obj[k]
+        if k.start < getattr(self._obj, "len", float("+Inf")):
+            return self._obj[k]
+
+        real_sect = self._elffile.get_section_addr(self._obj.sh_addr + k.start)
+        offs = self._obj.sh_addr - real_sect.sh_addr
+        if k.stop is str:
+            new_k = slice(k.start + offs, str)
+        else:
+            new_k = slice(k.start + offs, k.stop + offs)
+        return real_sect[new_k]
 
     def getreloc(self, offset):
         """
diff --git a/python/tsexpand.py b/python/tsexpand.py
new file mode 100644
index 0000000..5d60997
--- /dev/null
+++ b/python/tsexpand.py
@@ -0,0 +1,131 @@
+#!/usr/bin/python3
+# SPDX-License-Identifier: MIT
+#
+# 2024 by David Lamparter
+#
+# this tool edits an FRR source .c file to expand the typesafe DECLARE_DLIST
+# et al. definitions.  This can be helpful to get better warnings/errors from
+# GCC when something re. a typesafe container is involved.  You can also use
+# it on .h files.
+# The actual expansions created by this tool are written to separate files
+# called something like "lib/cspf__visited_tsexpand.h" (for a container named
+# "visited")
+#
+# THIS TOOL EDITS THE FILE IN PLACE.  MAKE A BACKUP IF YOU HAVE UNSAVED WORK
+# IN PROGRESS (which is likely because you're debugging a typesafe container
+# problem!)
+#
+# The PREDECL_XYZ is irrelevant for this tool, it needs to be run on the file
+# that has the DECLARE_XYZ (can be .c or .h)
+#
+# the lines added by this tool all have /* $ts_expand: remove$ */ at the end
+# you can undo the effects of this tool by calling sed:
+#
+#   sed -e '/\$ts_expand: remove\$/ d' -i.orig filename.c
+
+import os
+import sys
+import re
+import subprocess
+import shlex
+
+decl_re = re.compile(
+    r"""(?<=\n)[ \t]*DECLARE_(LIST|ATOMLIST|DLIST|HEAP|HASH|(SORTLIST|SKIPLIST|RBTREE|ATOMSORT)_(NON)?UNIQ)\(\s*(?P<name>[^, \t\n]+)\s*,[^)]+\)\s*;[ \t]*\n"""
+)
+kill_re = re.compile(r"""(?<=\n)[^\n]*/\* \$ts_expand: remove\$ \*/\n""")
+
+src_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+# some files may be compiled with different CPPFLAGS, that's not supported
+# here...
+cpp = subprocess.check_output(
+    ["make", "var-CPP", "var-AM_CPPFLAGS", "var-DEFS"], cwd=src_root
+)
+cpp = shlex.split(cpp.decode("UTF-8"))
+
+
+def process_file(filename):
+    with open(filename, "r") as ifd:
+        data = ifd.read()
+
+    data = kill_re.sub("", data)
+
+    before = 0
+
+    dirname = os.path.dirname(filename)
+    basename = os.path.basename(filename).removesuffix(".c").removesuffix(".h")
+
+    xname = filename + ".exp"
+    with open(filename + ".exp", "w") as ofd:
+        for m in decl_re.finditer(data):
+            s = m.start()
+            e = m.end()
+            ofd.write(data[before:s])
+
+            # start gcc/clang with some "magic" options to make it expand the
+            # typesafe macros, but nothing else.
+            #   -P removes the "#line" markers (which are useless because
+            #      everything ends up on one line anyway)
+            #   -D_TYPESAFE_EXPAND_MACROS prevents the system header files
+            #      (stddef.h, stdint.h, etc.) from being included and expanded
+            #   -imacros loads the macro definitions from typesafe.h, but
+            #      doesn't include any of the "plain text" (i.e. prototypes
+            #      and outside-macro struct definitions) from it
+            #   atomlist.h is sufficient because it includes typesafe.h which
+            #   includes typerb.h, that's all of them
+            p_expand = subprocess.Popen(
+                cpp
+                + [
+                    "-P",
+                    "-D_TYPESAFE_EXPAND_MACROS",
+                    "-imacros",
+                    "lib/atomlist.h",
+                    "-",
+                ],
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                cwd=src_root,
+            )
+            # the output will look like shit, all on one line.  format it.
+            p_format = subprocess.Popen(
+                ["clang-format", "-"],
+                stdin=p_expand.stdout,
+                stdout=subprocess.PIPE,
+                cwd=src_root,
+            )
+            # pipe between cpp & clang-format needs to be closed
+            p_expand.stdout.close()
+
+            # ... and finally, write the DECLARE_XYZ statement, and ONLY that
+            # statements.  No headers, no other definitions.
+            p_expand.stdin.write(data[s:e].encode("UTF-8"))
+            p_expand.stdin.close()
+
+            odata = b""
+            while rd := p_format.stdout.read():
+                odata = odata + rd
+
+            p_expand.wait()
+            p_format.wait()
+
+            # and now that we have the expanded text, write it out, put an
+            # #include in the .c file, and put "#if 0" around the original
+            # DECLARE_XYZ statement (otherwise it'll be duplicate...)
+            newname = os.path.join(dirname, f"{basename}__{m.group('name')}_tsexpand.h")
+            with open(newname, "wb") as nfd:
+                nfd.write(odata)
+
+            ofd.write(f'#include "{newname}" /* $ts_expand: remove$ */\n')
+            ofd.write("#if 0 /* $ts_expand: remove$ */\n")
+            ofd.write(data[s:e])
+            ofd.write("#endif /* $ts_expand: remove$ */\n")
+            before = e
+
+        ofd.write(data[before:])
+
+    os.rename(xname, filename)
+
+
+if __name__ == "__main__":
+    for filename in sys.argv[1:]:
+        process_file(filename)
diff --git a/python/xref2vtysh.py b/python/xref2vtysh.py
index 91ccba6..75d9ccf 100644
--- a/python/xref2vtysh.py
+++ b/python/xref2vtysh.py
@@ -32,11 +32,11 @@ frr_top_src = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 # not quite obvious...
 
 daemon_flags = {
-    "lib/agentx.c": "VTYSH_ISISD|VTYSH_RIPD|VTYSH_OSPFD|VTYSH_OSPF6D|VTYSH_BGPD|VTYSH_ZEBRA",
+    "lib/libagentx.c": "VTYSH_ISISD|VTYSH_RIPD|VTYSH_OSPFD|VTYSH_OSPF6D|VTYSH_BGPD|VTYSH_ZEBRA",
     "lib/filter.c": "VTYSH_ACL_SHOW",
     "lib/filter_cli.c": "VTYSH_ACL_CONFIG",
     "lib/if.c": "VTYSH_INTERFACE",
-    "lib/keychain.c": "VTYSH_KEYS",
+    "lib/keychain_cli.c": "VTYSH_KEYS",
     "lib/mgmt_be_client.c": "VTYSH_MGMT_BACKEND",
     "lib/mgmt_fe_client.c": "VTYSH_MGMT_FRONTEND",
     "lib/lib_vty.c": "VTYSH_ALL",
diff --git a/python/xrelfo.py b/python/xrelfo.py
index a40b19e..07cd740 100644
--- a/python/xrelfo.py
+++ b/python/xrelfo.py
@@ -22,7 +22,7 @@ import argparse
 
 from clippy.uidhash import uidhash
 from clippy.elf import *
-from clippy import frr_top_src, CmdAttr
+from clippy import frr_top_src, CmdAttr, elf_notes
 from tiabwarfo import FieldApplicator
 from xref2vtysh import CommandEntry
 
@@ -327,6 +327,7 @@ class Xrelfo(dict):
             }
         )
         self._xrefs = []
+        self.note_warn = False
 
     def load_file(self, filename):
         orig_filename = filename
@@ -395,6 +396,15 @@ class Xrelfo(dict):
             ptrs = edf.iter_data(XrefPtr, slice(start, end))
 
         else:
+            if elf_notes:
+                self.note_warn = True
+                sys.stderr.write(
+                    """%s: warning: binary has no FRRouting.XREF note
+%s-   one of FRR_MODULE_SETUP, FRR_DAEMON_INFO or XREF_SETUP must be used
+"""
+                    % (orig_filename, orig_filename)
+                )
+
             xrefarray = edf.get_section("xref_array")
             if xrefarray is None:
                 raise ValueError("file has neither xref note nor xref_array section")
@@ -471,6 +481,9 @@ def _main(args):
             sys.stderr.write("while processing %s:\n" % (fn))
             traceback.print_exc()
 
+    if xrelfo.note_warn and args.Werror:
+        errors += 1
+
     for option in dir(args):
         if option.startswith("W") and option != "Werror":
             checks = sorted(xrelfo.check(args))