summaryrefslogtreecommitdiffstats
path: root/tools/objtool
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 18:49:45 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 18:49:45 +0000
commit2c3c1048746a4622d8c89a29670120dc8fab93c4 (patch)
tree848558de17fb3008cdf4d861b01ac7781903ce39 /tools/objtool
parentInitial commit. (diff)
downloadlinux-2c3c1048746a4622d8c89a29670120dc8fab93c4.tar.xz
linux-2c3c1048746a4622d8c89a29670120dc8fab93c4.zip
Adding upstream version 6.1.76.upstream/6.1.76
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'tools/objtool')
-rw-r--r--tools/objtool/.gitignore4
-rw-r--r--tools/objtool/Build35
-rw-r--r--tools/objtool/Documentation/objtool.txt442
-rw-r--r--tools/objtool/Makefile70
-rw-r--r--tools/objtool/arch/x86/Build13
-rw-r--r--tools/objtool/arch/x86/decode.c806
-rw-r--r--tools/objtool/arch/x86/include/arch/cfi_regs.h25
-rw-r--r--tools/objtool/arch/x86/include/arch/elf.h6
-rw-r--r--tools/objtool/arch/x86/include/arch/endianness.h9
-rw-r--r--tools/objtool/arch/x86/include/arch/special.h21
-rw-r--r--tools/objtool/arch/x86/special.c145
-rw-r--r--tools/objtool/builtin-check.c212
-rw-r--r--tools/objtool/check.c4371
-rw-r--r--tools/objtool/elf.c1362
-rw-r--r--tools/objtool/include/objtool/arch.h97
-rw-r--r--tools/objtool/include/objtool/builtin.h46
-rw-r--r--tools/objtool/include/objtool/cfi.h40
-rw-r--r--tools/objtool/include/objtool/check.h109
-rw-r--r--tools/objtool/include/objtool/elf.h176
-rw-r--r--tools/objtool/include/objtool/endianness.h38
-rw-r--r--tools/objtool/include/objtool/objtool.h50
-rw-r--r--tools/objtool/include/objtool/special.h42
-rw-r--r--tools/objtool/include/objtool/warn.h67
-rw-r--r--tools/objtool/objtool.c150
-rw-r--r--tools/objtool/orc_dump.c215
-rw-r--r--tools/objtool/orc_gen.c245
-rw-r--r--tools/objtool/special.c181
-rwxr-xr-xtools/objtool/sync-check.sh75
-rw-r--r--tools/objtool/weak.c26
29 files changed, 9078 insertions, 0 deletions
diff --git a/tools/objtool/.gitignore b/tools/objtool/.gitignore
new file mode 100644
index 000000000..14236db36
--- /dev/null
+++ b/tools/objtool/.gitignore
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0-only
+arch/x86/lib/inat-tables.c
+/objtool
+fixdep
diff --git a/tools/objtool/Build b/tools/objtool/Build
new file mode 100644
index 000000000..33f2ee5a4
--- /dev/null
+++ b/tools/objtool/Build
@@ -0,0 +1,35 @@
+objtool-y += arch/$(SRCARCH)/
+
+objtool-y += weak.o
+
+objtool-y += check.o
+objtool-y += special.o
+objtool-y += builtin-check.o
+objtool-y += elf.o
+objtool-y += objtool.o
+
+objtool-$(BUILD_ORC) += orc_gen.o
+objtool-$(BUILD_ORC) += orc_dump.o
+
+objtool-y += libstring.o
+objtool-y += libctype.o
+objtool-y += str_error_r.o
+objtool-y += librbtree.o
+
+CFLAGS += -I$(srctree)/tools/lib
+
+$(OUTPUT)libstring.o: ../lib/string.c FORCE
+ $(call rule_mkdir)
+ $(call if_changed_dep,cc_o_c)
+
+$(OUTPUT)libctype.o: ../lib/ctype.c FORCE
+ $(call rule_mkdir)
+ $(call if_changed_dep,cc_o_c)
+
+$(OUTPUT)str_error_r.o: ../lib/str_error_r.c FORCE
+ $(call rule_mkdir)
+ $(call if_changed_dep,cc_o_c)
+
+$(OUTPUT)librbtree.o: ../lib/rbtree.c FORCE
+ $(call rule_mkdir)
+ $(call if_changed_dep,cc_o_c)
diff --git a/tools/objtool/Documentation/objtool.txt b/tools/objtool/Documentation/objtool.txt
new file mode 100644
index 000000000..8a671902a
--- /dev/null
+++ b/tools/objtool/Documentation/objtool.txt
@@ -0,0 +1,442 @@
+Objtool
+=======
+
+The kernel CONFIG_OBJTOOL option enables a host tool named 'objtool'
+which runs at compile time. It can do various validations and
+transformations on .o files.
+
+Objtool has become an integral part of the x86-64 kernel toolchain. The
+kernel depends on it for a variety of security and performance features
+(and other types of features as well).
+
+
+Features
+--------
+
+Objtool has the following features:
+
+- Stack unwinding metadata validation -- useful for helping to ensure
+ stack traces are reliable for live patching
+
+- ORC unwinder metadata generation -- a faster and more precise
+ alternative to frame pointer based unwinding
+
+- Retpoline validation -- ensures that all indirect calls go through
+ retpoline thunks, for Spectre v2 mitigations
+
+- Retpoline call site annotation -- annotates all retpoline thunk call
+ sites, enabling the kernel to patch them inline, to prevent "thunk
+ funneling" for both security and performance reasons
+
+- Non-instrumentation validation -- validates non-instrumentable
+ ("noinstr") code rules, preventing instrumentation in low-level C
+ entry code
+
+- Static call annotation -- annotates static call sites, enabling the
+ kernel to implement inline static calls, a faster alternative to some
+ indirect branches
+
+- Uaccess validation -- validates uaccess rules for a proper
+ implementation of Supervisor Mode Access Protection (SMAP)
+
+- Straight Line Speculation validation -- validates certain SLS
+ mitigations
+
+- Indirect Branch Tracking validation -- validates Intel CET IBT rules
+ to ensure that all functions referenced by function pointers have
+ corresponding ENDBR instructions
+
+- Indirect Branch Tracking annotation -- annotates unused ENDBR
+ instruction sites, enabling the kernel to "seal" them (replace them
+ with NOPs) to further harden IBT
+
+- Function entry annotation -- annotates function entries, enabling
+ kernel function tracing
+
+- Other toolchain hacks which will go unmentioned at this time...
+
+Each feature can be enabled individually or in combination using the
+objtool cmdline.
+
+
+Objects
+-------
+
+Typically, objtool runs on every translation unit (TU, aka ".o file") in
+the kernel. If a TU is part of a kernel module, the '--module' option
+is added.
+
+However:
+
+- If noinstr validation is enabled, it also runs on vmlinux.o, with all
+ options removed and '--noinstr' added.
+
+- If IBT or LTO is enabled, it doesn't run on TUs at all. Instead it
+ runs on vmlinux.o and linked modules, with all options.
+
+In summary:
+
+ A) Legacy mode:
+ TU: objtool [--module] <options>
+ vmlinux: N/A
+ module: N/A
+
+ B) CONFIG_NOINSTR_VALIDATION=y && !(CONFIG_X86_KERNEL_IBT=y || CONFIG_LTO=y):
+ TU: objtool [--module] <options> // no --noinstr
+ vmlinux: objtool --noinstr // other options removed
+ module: N/A
+
+ C) CONFIG_X86_KERNEL_IBT=y || CONFIG_LTO=y:
+ TU: N/A
+ vmlinux: objtool --noinstr <options>
+ module: objtool --module --noinstr <options>
+
+
+Stack validation
+----------------
+
+Objtool's stack validation feature analyzes every .o file and ensures
+the validity of its stack metadata. It enforces a set of rules on asm
+code and C inline assembly code so that stack traces can be reliable.
+
+For each function, it recursively follows all possible code paths and
+validates the correct frame pointer state at each instruction.
+
+It also follows code paths involving special sections, like
+.altinstructions, __jump_table, and __ex_table, which can add
+alternative execution paths to a given instruction (or set of
+instructions). Similarly, it knows how to follow switch statements, for
+which gcc sometimes uses jump tables.
+
+Here are some of the benefits of validating stack metadata:
+
+a) More reliable stack traces for frame pointer enabled kernels
+
+ Frame pointers are used for debugging purposes. They allow runtime
+ code and debug tools to be able to walk the stack to determine the
+ chain of function call sites that led to the currently executing
+ code.
+
+ For some architectures, frame pointers are enabled by
+ CONFIG_FRAME_POINTER. For some other architectures they may be
+ required by the ABI (sometimes referred to as "backchain pointers").
+
+ For C code, gcc automatically generates instructions for setting up
+ frame pointers when the -fno-omit-frame-pointer option is used.
+
+ But for asm code, the frame setup instructions have to be written by
+ hand, which most people don't do. So the end result is that
+ CONFIG_FRAME_POINTER is honored for C code but not for most asm code.
+
+ For stack traces based on frame pointers to be reliable, all
+ functions which call other functions must first create a stack frame
+ and update the frame pointer. If a first function doesn't properly
+ create a stack frame before calling a second function, the *caller*
+ of the first function will be skipped on the stack trace.
+
+ For example, consider the following example backtrace with frame
+ pointers enabled:
+
+ [<ffffffff81812584>] dump_stack+0x4b/0x63
+ [<ffffffff812d6dc2>] cmdline_proc_show+0x12/0x30
+ [<ffffffff8127f568>] seq_read+0x108/0x3e0
+ [<ffffffff812cce62>] proc_reg_read+0x42/0x70
+ [<ffffffff81256197>] __vfs_read+0x37/0x100
+ [<ffffffff81256b16>] vfs_read+0x86/0x130
+ [<ffffffff81257898>] SyS_read+0x58/0xd0
+ [<ffffffff8181c1f2>] entry_SYSCALL_64_fastpath+0x12/0x76
+
+ It correctly shows that the caller of cmdline_proc_show() is
+ seq_read().
+
+ If we remove the frame pointer logic from cmdline_proc_show() by
+ replacing the frame pointer related instructions with nops, here's
+ what it looks like instead:
+
+ [<ffffffff81812584>] dump_stack+0x4b/0x63
+ [<ffffffff812d6dc2>] cmdline_proc_show+0x12/0x30
+ [<ffffffff812cce62>] proc_reg_read+0x42/0x70
+ [<ffffffff81256197>] __vfs_read+0x37/0x100
+ [<ffffffff81256b16>] vfs_read+0x86/0x130
+ [<ffffffff81257898>] SyS_read+0x58/0xd0
+ [<ffffffff8181c1f2>] entry_SYSCALL_64_fastpath+0x12/0x76
+
+ Notice that cmdline_proc_show()'s caller, seq_read(), has been
+ skipped. Instead the stack trace seems to show that
+ cmdline_proc_show() was called by proc_reg_read().
+
+ The benefit of objtool here is that because it ensures that *all*
+ functions honor CONFIG_FRAME_POINTER, no functions will ever[*] be
+ skipped on a stack trace.
+
+ [*] unless an interrupt or exception has occurred at the very
+ beginning of a function before the stack frame has been created,
+ or at the very end of the function after the stack frame has been
+ destroyed. This is an inherent limitation of frame pointers.
+
+b) ORC (Oops Rewind Capability) unwind table generation
+
+ An alternative to frame pointers and DWARF, ORC unwind data can be
+ used to walk the stack. Unlike frame pointers, ORC data is out of
+ band. So it doesn't affect runtime performance and it can be
+ reliable even when interrupts or exceptions are involved.
+
+ For more details, see Documentation/x86/orc-unwinder.rst.
+
+c) Higher live patching compatibility rate
+
+ Livepatch has an optional "consistency model", which is needed for
+ more complex patches. In order for the consistency model to work,
+ stack traces need to be reliable (or an unreliable condition needs to
+ be detectable). Objtool makes that possible.
+
+ For more details, see the livepatch documentation in the Linux kernel
+ source tree at Documentation/livepatch/livepatch.rst.
+
+To achieve the validation, objtool enforces the following rules:
+
+1. Each callable function must be annotated as such with the ELF
+ function type. In asm code, this is typically done using the
+ ENTRY/ENDPROC macros. If objtool finds a return instruction
+ outside of a function, it flags an error since that usually indicates
+ callable code which should be annotated accordingly.
+
+ This rule is needed so that objtool can properly identify each
+ callable function in order to analyze its stack metadata.
+
+2. Conversely, each section of code which is *not* callable should *not*
+ be annotated as an ELF function. The ENDPROC macro shouldn't be used
+ in this case.
+
+ This rule is needed so that objtool can ignore non-callable code.
+ Such code doesn't have to follow any of the other rules.
+
+3. Each callable function which calls another function must have the
+ correct frame pointer logic, if required by CONFIG_FRAME_POINTER or
+ the architecture's back chain rules. This can by done in asm code
+ with the FRAME_BEGIN/FRAME_END macros.
+
+ This rule ensures that frame pointer based stack traces will work as
+ designed. If function A doesn't create a stack frame before calling
+ function B, the _caller_ of function A will be skipped on the stack
+ trace.
+
+4. Dynamic jumps and jumps to undefined symbols are only allowed if:
+
+ a) the jump is part of a switch statement; or
+
+ b) the jump matches sibling call semantics and the frame pointer has
+ the same value it had on function entry.
+
+ This rule is needed so that objtool can reliably analyze all of a
+ function's code paths. If a function jumps to code in another file,
+ and it's not a sibling call, objtool has no way to follow the jump
+ because it only analyzes a single file at a time.
+
+5. A callable function may not execute kernel entry/exit instructions.
+ The only code which needs such instructions is kernel entry code,
+ which shouldn't be be in callable functions anyway.
+
+ This rule is just a sanity check to ensure that callable functions
+ return normally.
+
+
+Objtool warnings
+----------------
+
+For asm files, if you're getting an error which doesn't make sense,
+first make sure that the affected code follows the above rules.
+
+For C files, the common culprits are inline asm statements and calls to
+"noreturn" functions. See below for more details.
+
+Another possible cause for errors in C code is if the Makefile removes
+-fno-omit-frame-pointer or adds -fomit-frame-pointer to the gcc options.
+
+Here are some examples of common warnings reported by objtool, what
+they mean, and suggestions for how to fix them. When in doubt, ping
+the objtool maintainers.
+
+
+1. file.o: warning: objtool: func()+0x128: call without frame pointer save/setup
+
+ The func() function made a function call without first saving and/or
+ updating the frame pointer, and CONFIG_FRAME_POINTER is enabled.
+
+ If the error is for an asm file, and func() is indeed a callable
+ function, add proper frame pointer logic using the FRAME_BEGIN and
+ FRAME_END macros. Otherwise, if it's not a callable function, remove
+ its ELF function annotation by changing ENDPROC to END, and instead
+ use the manual unwind hint macros in asm/unwind_hints.h.
+
+ If it's a GCC-compiled .c file, the error may be because the function
+ uses an inline asm() statement which has a "call" instruction. An
+ asm() statement with a call instruction must declare the use of the
+ stack pointer in its output operand. On x86_64, this means adding
+ the ASM_CALL_CONSTRAINT as an output constraint:
+
+ asm volatile("call func" : ASM_CALL_CONSTRAINT);
+
+ Otherwise the stack frame may not get created before the call.
+
+
+2. file.o: warning: objtool: .text+0x53: unreachable instruction
+
+ Objtool couldn't find a code path to reach the instruction.
+
+ If the error is for an asm file, and the instruction is inside (or
+ reachable from) a callable function, the function should be annotated
+ with the ENTRY/ENDPROC macros (ENDPROC is the important one).
+ Otherwise, the code should probably be annotated with the unwind hint
+ macros in asm/unwind_hints.h so objtool and the unwinder can know the
+ stack state associated with the code.
+
+ If you're 100% sure the code won't affect stack traces, or if you're
+ a just a bad person, you can tell objtool to ignore it. See the
+ "Adding exceptions" section below.
+
+ If it's not actually in a callable function (e.g. kernel entry code),
+ change ENDPROC to END.
+
+
+4. file.o: warning: objtool: func(): can't find starting instruction
+ or
+ file.o: warning: objtool: func()+0x11dd: can't decode instruction
+
+ Does the file have data in a text section? If so, that can confuse
+ objtool's instruction decoder. Move the data to a more appropriate
+ section like .data or .rodata.
+
+
+5. file.o: warning: objtool: func()+0x6: unsupported instruction in callable function
+
+ This is a kernel entry/exit instruction like sysenter or iret. Such
+ instructions aren't allowed in a callable function, and are most
+ likely part of the kernel entry code. They should usually not have
+ the callable function annotation (ENDPROC) and should always be
+ annotated with the unwind hint macros in asm/unwind_hints.h.
+
+
+6. file.o: warning: objtool: func()+0x26: sibling call from callable instruction with modified stack frame
+
+ This is a dynamic jump or a jump to an undefined symbol. Objtool
+ assumed it's a sibling call and detected that the frame pointer
+ wasn't first restored to its original state.
+
+ If it's not really a sibling call, you may need to move the
+ destination code to the local file.
+
+ If the instruction is not actually in a callable function (e.g.
+ kernel entry code), change ENDPROC to END and annotate manually with
+ the unwind hint macros in asm/unwind_hints.h.
+
+
+7. file: warning: objtool: func()+0x5c: stack state mismatch
+
+ The instruction's frame pointer state is inconsistent, depending on
+ which execution path was taken to reach the instruction.
+
+ Make sure that, when CONFIG_FRAME_POINTER is enabled, the function
+ pushes and sets up the frame pointer (for x86_64, this means rbp) at
+ the beginning of the function and pops it at the end of the function.
+ Also make sure that no other code in the function touches the frame
+ pointer.
+
+ Another possibility is that the code has some asm or inline asm which
+ does some unusual things to the stack or the frame pointer. In such
+ cases it's probably appropriate to use the unwind hint macros in
+ asm/unwind_hints.h.
+
+
+8. file.o: warning: objtool: funcA() falls through to next function funcB()
+
+ This means that funcA() doesn't end with a return instruction or an
+ unconditional jump, and that objtool has determined that the function
+ can fall through into the next function. There could be different
+ reasons for this:
+
+ 1) funcA()'s last instruction is a call to a "noreturn" function like
+ panic(). In this case the noreturn function needs to be added to
+ objtool's hard-coded global_noreturns array. Feel free to bug the
+ objtool maintainer, or you can submit a patch.
+
+ 2) funcA() uses the unreachable() annotation in a section of code
+ that is actually reachable.
+
+ 3) If funcA() calls an inline function, the object code for funcA()
+ might be corrupt due to a gcc bug. For more details, see:
+ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70646
+
+9. file.o: warning: objtool: funcA() call to funcB() with UACCESS enabled
+
+ This means that an unexpected call to a non-whitelisted function exists
+ outside of arch-specific guards.
+ X86: SMAP (stac/clac): __uaccess_begin()/__uaccess_end()
+ ARM: PAN: uaccess_enable()/uaccess_disable()
+
+ These functions should be called to denote a minimal critical section around
+ access to __user variables. See also: https://lwn.net/Articles/517475/
+
+ The intention of the warning is to prevent calls to funcB() from eventually
+ calling schedule(), potentially leaking the AC flags state, and not
+ restoring them correctly.
+
+ It also helps verify that there are no unexpected calls to funcB() which may
+ access user space pages with protections against doing so disabled.
+
+ To fix, either:
+ 1) remove explicit calls to funcB() from funcA().
+ 2) add the correct guards before and after calls to low level functions like
+ __get_user_size()/__put_user_size().
+ 3) add funcB to uaccess_safe_builtin whitelist in tools/objtool/check.c, if
+ funcB obviously does not call schedule(), and is marked notrace (since
+ function tracing inserts additional calls, which is not obvious from the
+ sources).
+
+10. file.o: warning: func()+0x5c: stack layout conflict in alternatives
+
+ This means that in the use of the alternative() or ALTERNATIVE()
+ macro, the code paths have conflicting modifications to the stack.
+ The problem is that there is only one ORC unwind table, which means
+ that the ORC unwind entries must be consistent for all possible
+ instruction boundaries regardless of which code has been patched.
+ This limitation can be overcome by massaging the alternatives with
+ NOPs to shift the stack changes around so they no longer conflict.
+
+11. file.o: warning: unannotated intra-function call
+
+ This warning means that a direct call is done to a destination which
+ is not at the beginning of a function. If this is a legit call, you
+ can remove this warning by putting the ANNOTATE_INTRA_FUNCTION_CALL
+ directive right before the call.
+
+
+If the error doesn't seem to make sense, it could be a bug in objtool.
+Feel free to ask the objtool maintainer for help.
+
+
+Adding exceptions
+-----------------
+
+If you _really_ need objtool to ignore something, and are 100% sure
+that it won't affect kernel stack traces, you can tell objtool to
+ignore it:
+
+- To skip validation of a function, use the STACK_FRAME_NON_STANDARD
+ macro.
+
+- To skip validation of a file, add
+
+ OBJECT_FILES_NON_STANDARD_filename.o := y
+
+ to the Makefile.
+
+- To skip validation of a directory, add
+
+ OBJECT_FILES_NON_STANDARD := y
+
+ to the Makefile.
+
+NOTE: OBJECT_FILES_NON_STANDARD doesn't work for link time validation of
+vmlinux.o or a linked module. So it should only be used for files which
+aren't linked into vmlinux or a module.
diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile
new file mode 100644
index 000000000..a3a9cc24e
--- /dev/null
+++ b/tools/objtool/Makefile
@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: GPL-2.0
+include ../scripts/Makefile.include
+include ../scripts/Makefile.arch
+
+# always use the host compiler
+AR = $(HOSTAR)
+CC = $(HOSTCC)
+LD = $(HOSTLD)
+
+ifeq ($(srctree),)
+srctree := $(patsubst %/,%,$(dir $(CURDIR)))
+srctree := $(patsubst %/,%,$(dir $(srctree)))
+endif
+
+SUBCMD_SRCDIR = $(srctree)/tools/lib/subcmd/
+LIBSUBCMD_OUTPUT = $(or $(OUTPUT),$(CURDIR)/)
+LIBSUBCMD = $(LIBSUBCMD_OUTPUT)libsubcmd.a
+
+OBJTOOL := $(OUTPUT)objtool
+OBJTOOL_IN := $(OBJTOOL)-in.o
+
+LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
+LIBELF_LIBS := $(shell $(HOSTPKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf)
+
+all: $(OBJTOOL)
+
+INCLUDES := -I$(srctree)/tools/include \
+ -I$(srctree)/tools/arch/$(HOSTARCH)/include/uapi \
+ -I$(srctree)/tools/arch/$(SRCARCH)/include \
+ -I$(srctree)/tools/objtool/include \
+ -I$(srctree)/tools/objtool/arch/$(SRCARCH)/include
+WARNINGS := $(EXTRA_WARNINGS) -Wno-switch-default -Wno-switch-enum -Wno-packed -Wno-nested-externs
+CFLAGS := -Werror $(WARNINGS) $(KBUILD_HOSTCFLAGS) -g $(INCLUDES) $(LIBELF_FLAGS)
+LDFLAGS += $(LIBELF_LIBS) $(LIBSUBCMD) $(KBUILD_HOSTLDFLAGS)
+
+# Allow old libelf to be used:
+elfshdr := $(shell echo '$(pound)include <libelf.h>' | $(CC) $(CFLAGS) -x c -E - | grep elf_getshdr)
+CFLAGS += $(if $(elfshdr),,-DLIBELF_USE_DEPRECATED)
+
+AWK = awk
+
+BUILD_ORC := n
+
+ifeq ($(SRCARCH),x86)
+ BUILD_ORC := y
+endif
+
+export BUILD_ORC
+export srctree OUTPUT CFLAGS SRCARCH AWK
+include $(srctree)/tools/build/Makefile.include
+
+$(OBJTOOL_IN): fixdep FORCE
+ @$(CONFIG_SHELL) ./sync-check.sh
+ @$(MAKE) $(build)=objtool
+
+$(OBJTOOL): $(LIBSUBCMD) $(OBJTOOL_IN)
+ $(QUIET_LINK)$(CC) $(OBJTOOL_IN) $(LDFLAGS) -o $@
+
+
+$(LIBSUBCMD): fixdep FORCE
+ $(Q)$(MAKE) -C $(SUBCMD_SRCDIR) OUTPUT=$(LIBSUBCMD_OUTPUT)
+
+clean:
+ $(call QUIET_CLEAN, objtool) $(RM) $(OBJTOOL)
+ $(Q)find $(OUTPUT) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete
+ $(Q)$(RM) $(OUTPUT)arch/x86/lib/inat-tables.c $(OUTPUT)fixdep $(LIBSUBCMD)
+
+FORCE:
+
+.PHONY: clean FORCE
diff --git a/tools/objtool/arch/x86/Build b/tools/objtool/arch/x86/Build
new file mode 100644
index 000000000..9f7869b5c
--- /dev/null
+++ b/tools/objtool/arch/x86/Build
@@ -0,0 +1,13 @@
+objtool-y += special.o
+objtool-y += decode.o
+
+inat_tables_script = ../arch/x86/tools/gen-insn-attr-x86.awk
+inat_tables_maps = ../arch/x86/lib/x86-opcode-map.txt
+
+$(OUTPUT)arch/x86/lib/inat-tables.c: $(inat_tables_script) $(inat_tables_maps)
+ $(call rule_mkdir)
+ $(Q)$(call echo-cmd,gen)$(AWK) -f $(inat_tables_script) $(inat_tables_maps) > $@
+
+$(OUTPUT)arch/x86/decode.o: $(OUTPUT)arch/x86/lib/inat-tables.c
+
+CFLAGS_decode.o += -I$(OUTPUT)arch/x86/lib
diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c
new file mode 100644
index 000000000..1ed49ab4e
--- /dev/null
+++ b/tools/objtool/arch/x86/decode.c
@@ -0,0 +1,806 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com>
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define unlikely(cond) (cond)
+#include <asm/insn.h>
+#include "../../../arch/x86/lib/inat.c"
+#include "../../../arch/x86/lib/insn.c"
+
+#define CONFIG_64BIT 1
+#include <asm/nops.h>
+
+#include <asm/orc_types.h>
+#include <objtool/check.h>
+#include <objtool/elf.h>
+#include <objtool/arch.h>
+#include <objtool/warn.h>
+#include <objtool/endianness.h>
+#include <objtool/builtin.h>
+#include <arch/elf.h>
+
+static int is_x86_64(const struct elf *elf)
+{
+ switch (elf->ehdr.e_machine) {
+ case EM_X86_64:
+ return 1;
+ case EM_386:
+ return 0;
+ default:
+ WARN("unexpected ELF machine type %d", elf->ehdr.e_machine);
+ return -1;
+ }
+}
+
+bool arch_callee_saved_reg(unsigned char reg)
+{
+ switch (reg) {
+ case CFI_BP:
+ case CFI_BX:
+ case CFI_R12:
+ case CFI_R13:
+ case CFI_R14:
+ case CFI_R15:
+ return true;
+
+ case CFI_AX:
+ case CFI_CX:
+ case CFI_DX:
+ case CFI_SI:
+ case CFI_DI:
+ case CFI_SP:
+ case CFI_R8:
+ case CFI_R9:
+ case CFI_R10:
+ case CFI_R11:
+ case CFI_RA:
+ default:
+ return false;
+ }
+}
+
+unsigned long arch_dest_reloc_offset(int addend)
+{
+ return addend + 4;
+}
+
+unsigned long arch_jump_destination(struct instruction *insn)
+{
+ return insn->offset + insn->len + insn->immediate;
+}
+
+#define ADD_OP(op) \
+ if (!(op = calloc(1, sizeof(*op)))) \
+ return -1; \
+ else for (list_add_tail(&op->list, ops_list); op; op = NULL)
+
+/*
+ * Helpers to decode ModRM/SIB:
+ *
+ * r/m| AX CX DX BX | SP | BP | SI DI |
+ * | R8 R9 R10 R11 | R12 | R13 | R14 R15 |
+ * Mod+----------------+-----+-----+---------+
+ * 00 | [r/m] |[SIB]|[IP+]| [r/m] |
+ * 01 | [r/m + d8] |[S+d]| [r/m + d8] |
+ * 10 | [r/m + d32] |[S+D]| [r/m + d32] |
+ * 11 | r/ m |
+ */
+
+#define mod_is_mem() (modrm_mod != 3)
+#define mod_is_reg() (modrm_mod == 3)
+
+#define is_RIP() ((modrm_rm & 7) == CFI_BP && modrm_mod == 0)
+#define have_SIB() ((modrm_rm & 7) == CFI_SP && mod_is_mem())
+
+#define rm_is(reg) (have_SIB() ? \
+ sib_base == (reg) && sib_index == CFI_SP : \
+ modrm_rm == (reg))
+
+#define rm_is_mem(reg) (mod_is_mem() && !is_RIP() && rm_is(reg))
+#define rm_is_reg(reg) (mod_is_reg() && modrm_rm == (reg))
+
+static bool has_notrack_prefix(struct insn *insn)
+{
+ int i;
+
+ for (i = 0; i < insn->prefixes.nbytes; i++) {
+ if (insn->prefixes.bytes[i] == 0x3e)
+ return true;
+ }
+
+ return false;
+}
+
+int arch_decode_instruction(struct objtool_file *file, const struct section *sec,
+ unsigned long offset, unsigned int maxlen,
+ unsigned int *len, enum insn_type *type,
+ unsigned long *immediate,
+ struct list_head *ops_list)
+{
+ const struct elf *elf = file->elf;
+ struct insn insn;
+ int x86_64, ret;
+ unsigned char op1, op2, op3, prefix,
+ rex = 0, rex_b = 0, rex_r = 0, rex_w = 0, rex_x = 0,
+ modrm = 0, modrm_mod = 0, modrm_rm = 0, modrm_reg = 0,
+ sib = 0, /* sib_scale = 0, */ sib_index = 0, sib_base = 0;
+ struct stack_op *op = NULL;
+ struct symbol *sym;
+ u64 imm;
+
+ x86_64 = is_x86_64(elf);
+ if (x86_64 == -1)
+ return -1;
+
+ ret = insn_decode(&insn, sec->data->d_buf + offset, maxlen,
+ x86_64 ? INSN_MODE_64 : INSN_MODE_32);
+ if (ret < 0) {
+ WARN("can't decode instruction at %s:0x%lx", sec->name, offset);
+ return -1;
+ }
+
+ *len = insn.length;
+ *type = INSN_OTHER;
+
+ if (insn.vex_prefix.nbytes)
+ return 0;
+
+ prefix = insn.prefixes.bytes[0];
+
+ op1 = insn.opcode.bytes[0];
+ op2 = insn.opcode.bytes[1];
+ op3 = insn.opcode.bytes[2];
+
+ if (insn.rex_prefix.nbytes) {
+ rex = insn.rex_prefix.bytes[0];
+ rex_w = X86_REX_W(rex) >> 3;
+ rex_r = X86_REX_R(rex) >> 2;
+ rex_x = X86_REX_X(rex) >> 1;
+ rex_b = X86_REX_B(rex);
+ }
+
+ if (insn.modrm.nbytes) {
+ modrm = insn.modrm.bytes[0];
+ modrm_mod = X86_MODRM_MOD(modrm);
+ modrm_reg = X86_MODRM_REG(modrm) + 8*rex_r;
+ modrm_rm = X86_MODRM_RM(modrm) + 8*rex_b;
+ }
+
+ if (insn.sib.nbytes) {
+ sib = insn.sib.bytes[0];
+ /* sib_scale = X86_SIB_SCALE(sib); */
+ sib_index = X86_SIB_INDEX(sib) + 8*rex_x;
+ sib_base = X86_SIB_BASE(sib) + 8*rex_b;
+ }
+
+ switch (op1) {
+
+ case 0x1:
+ case 0x29:
+ if (rex_w && rm_is_reg(CFI_SP)) {
+
+ /* add/sub reg, %rsp */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_ADD;
+ op->src.reg = modrm_reg;
+ op->dest.type = OP_DEST_REG;
+ op->dest.reg = CFI_SP;
+ }
+ }
+ break;
+
+ case 0x50 ... 0x57:
+
+ /* push reg */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_REG;
+ op->src.reg = (op1 & 0x7) + 8*rex_b;
+ op->dest.type = OP_DEST_PUSH;
+ }
+
+ break;
+
+ case 0x58 ... 0x5f:
+
+ /* pop reg */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_POP;
+ op->dest.type = OP_DEST_REG;
+ op->dest.reg = (op1 & 0x7) + 8*rex_b;
+ }
+
+ break;
+
+ case 0x68:
+ case 0x6a:
+ /* push immediate */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_CONST;
+ op->dest.type = OP_DEST_PUSH;
+ }
+ break;
+
+ case 0x70 ... 0x7f:
+ *type = INSN_JUMP_CONDITIONAL;
+ break;
+
+ case 0x80 ... 0x83:
+ /*
+ * 1000 00sw : mod OP r/m : immediate
+ *
+ * s - sign extend immediate
+ * w - imm8 / imm32
+ *
+ * OP: 000 ADD 100 AND
+ * 001 OR 101 SUB
+ * 010 ADC 110 XOR
+ * 011 SBB 111 CMP
+ */
+
+ /* 64bit only */
+ if (!rex_w)
+ break;
+
+ /* %rsp target only */
+ if (!rm_is_reg(CFI_SP))
+ break;
+
+ imm = insn.immediate.value;
+ if (op1 & 2) { /* sign extend */
+ if (op1 & 1) { /* imm32 */
+ imm <<= 32;
+ imm = (s64)imm >> 32;
+ } else { /* imm8 */
+ imm <<= 56;
+ imm = (s64)imm >> 56;
+ }
+ }
+
+ switch (modrm_reg & 7) {
+ case 5:
+ imm = -imm;
+ /* fallthrough */
+ case 0:
+ /* add/sub imm, %rsp */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_ADD;
+ op->src.reg = CFI_SP;
+ op->src.offset = imm;
+ op->dest.type = OP_DEST_REG;
+ op->dest.reg = CFI_SP;
+ }
+ break;
+
+ case 4:
+ /* and imm, %rsp */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_AND;
+ op->src.reg = CFI_SP;
+ op->src.offset = insn.immediate.value;
+ op->dest.type = OP_DEST_REG;
+ op->dest.reg = CFI_SP;
+ }
+ break;
+
+ default:
+ /* WARN ? */
+ break;
+ }
+
+ break;
+
+ case 0x89:
+ if (!rex_w)
+ break;
+
+ if (modrm_reg == CFI_SP) {
+
+ if (mod_is_reg()) {
+ /* mov %rsp, reg */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_REG;
+ op->src.reg = CFI_SP;
+ op->dest.type = OP_DEST_REG;
+ op->dest.reg = modrm_rm;
+ }
+ break;
+
+ } else {
+ /* skip RIP relative displacement */
+ if (is_RIP())
+ break;
+
+ /* skip nontrivial SIB */
+ if (have_SIB()) {
+ modrm_rm = sib_base;
+ if (sib_index != CFI_SP)
+ break;
+ }
+
+ /* mov %rsp, disp(%reg) */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_REG;
+ op->src.reg = CFI_SP;
+ op->dest.type = OP_DEST_REG_INDIRECT;
+ op->dest.reg = modrm_rm;
+ op->dest.offset = insn.displacement.value;
+ }
+ break;
+ }
+
+ break;
+ }
+
+ if (rm_is_reg(CFI_SP)) {
+
+ /* mov reg, %rsp */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_REG;
+ op->src.reg = modrm_reg;
+ op->dest.type = OP_DEST_REG;
+ op->dest.reg = CFI_SP;
+ }
+ break;
+ }
+
+ /* fallthrough */
+ case 0x88:
+ if (!rex_w)
+ break;
+
+ if (rm_is_mem(CFI_BP)) {
+
+ /* mov reg, disp(%rbp) */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_REG;
+ op->src.reg = modrm_reg;
+ op->dest.type = OP_DEST_REG_INDIRECT;
+ op->dest.reg = CFI_BP;
+ op->dest.offset = insn.displacement.value;
+ }
+ break;
+ }
+
+ if (rm_is_mem(CFI_SP)) {
+
+ /* mov reg, disp(%rsp) */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_REG;
+ op->src.reg = modrm_reg;
+ op->dest.type = OP_DEST_REG_INDIRECT;
+ op->dest.reg = CFI_SP;
+ op->dest.offset = insn.displacement.value;
+ }
+ break;
+ }
+
+ break;
+
+ case 0x8b:
+ if (!rex_w)
+ break;
+
+ if (rm_is_mem(CFI_BP)) {
+
+ /* mov disp(%rbp), reg */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_REG_INDIRECT;
+ op->src.reg = CFI_BP;
+ op->src.offset = insn.displacement.value;
+ op->dest.type = OP_DEST_REG;
+ op->dest.reg = modrm_reg;
+ }
+ break;
+ }
+
+ if (rm_is_mem(CFI_SP)) {
+
+ /* mov disp(%rsp), reg */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_REG_INDIRECT;
+ op->src.reg = CFI_SP;
+ op->src.offset = insn.displacement.value;
+ op->dest.type = OP_DEST_REG;
+ op->dest.reg = modrm_reg;
+ }
+ break;
+ }
+
+ break;
+
+ case 0x8d:
+ if (mod_is_reg()) {
+ WARN("invalid LEA encoding at %s:0x%lx", sec->name, offset);
+ break;
+ }
+
+ /* skip non 64bit ops */
+ if (!rex_w)
+ break;
+
+ /* skip RIP relative displacement */
+ if (is_RIP())
+ break;
+
+ /* skip nontrivial SIB */
+ if (have_SIB()) {
+ modrm_rm = sib_base;
+ if (sib_index != CFI_SP)
+ break;
+ }
+
+ /* lea disp(%src), %dst */
+ ADD_OP(op) {
+ op->src.offset = insn.displacement.value;
+ if (!op->src.offset) {
+ /* lea (%src), %dst */
+ op->src.type = OP_SRC_REG;
+ } else {
+ /* lea disp(%src), %dst */
+ op->src.type = OP_SRC_ADD;
+ }
+ op->src.reg = modrm_rm;
+ op->dest.type = OP_DEST_REG;
+ op->dest.reg = modrm_reg;
+ }
+ break;
+
+ case 0x8f:
+ /* pop to mem */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_POP;
+ op->dest.type = OP_DEST_MEM;
+ }
+ break;
+
+ case 0x90:
+ *type = INSN_NOP;
+ break;
+
+ case 0x9c:
+ /* pushf */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_CONST;
+ op->dest.type = OP_DEST_PUSHF;
+ }
+ break;
+
+ case 0x9d:
+ /* popf */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_POPF;
+ op->dest.type = OP_DEST_MEM;
+ }
+ break;
+
+ case 0x0f:
+
+ if (op2 == 0x01) {
+
+ if (modrm == 0xca)
+ *type = INSN_CLAC;
+ else if (modrm == 0xcb)
+ *type = INSN_STAC;
+
+ } else if (op2 >= 0x80 && op2 <= 0x8f) {
+
+ *type = INSN_JUMP_CONDITIONAL;
+
+ } else if (op2 == 0x05 || op2 == 0x07 || op2 == 0x34 ||
+ op2 == 0x35) {
+
+ /* sysenter, sysret */
+ *type = INSN_CONTEXT_SWITCH;
+
+ } else if (op2 == 0x0b || op2 == 0xb9) {
+
+ /* ud2 */
+ *type = INSN_BUG;
+
+ } else if (op2 == 0x0d || op2 == 0x1f) {
+
+ /* nopl/nopw */
+ *type = INSN_NOP;
+
+ } else if (op2 == 0x1e) {
+
+ if (prefix == 0xf3 && (modrm == 0xfa || modrm == 0xfb))
+ *type = INSN_ENDBR;
+
+
+ } else if (op2 == 0x38 && op3 == 0xf8) {
+ if (insn.prefixes.nbytes == 1 &&
+ insn.prefixes.bytes[0] == 0xf2) {
+ /* ENQCMD cannot be used in the kernel. */
+ WARN("ENQCMD instruction at %s:%lx", sec->name,
+ offset);
+ }
+
+ } else if (op2 == 0xa0 || op2 == 0xa8) {
+
+ /* push fs/gs */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_CONST;
+ op->dest.type = OP_DEST_PUSH;
+ }
+
+ } else if (op2 == 0xa1 || op2 == 0xa9) {
+
+ /* pop fs/gs */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_POP;
+ op->dest.type = OP_DEST_MEM;
+ }
+ }
+
+ break;
+
+ case 0xc9:
+ /*
+ * leave
+ *
+ * equivalent to:
+ * mov bp, sp
+ * pop bp
+ */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_REG;
+ op->src.reg = CFI_BP;
+ op->dest.type = OP_DEST_REG;
+ op->dest.reg = CFI_SP;
+ }
+ ADD_OP(op) {
+ op->src.type = OP_SRC_POP;
+ op->dest.type = OP_DEST_REG;
+ op->dest.reg = CFI_BP;
+ }
+ break;
+
+ case 0xcc:
+ /* int3 */
+ *type = INSN_TRAP;
+ break;
+
+ case 0xe3:
+ /* jecxz/jrcxz */
+ *type = INSN_JUMP_CONDITIONAL;
+ break;
+
+ case 0xe9:
+ case 0xeb:
+ *type = INSN_JUMP_UNCONDITIONAL;
+ break;
+
+ case 0xc2:
+ case 0xc3:
+ *type = INSN_RETURN;
+ break;
+
+ case 0xc7: /* mov imm, r/m */
+ if (!opts.noinstr)
+ break;
+
+ if (insn.length == 3+4+4 && !strncmp(sec->name, ".init.text", 10)) {
+ struct reloc *immr, *disp;
+ struct symbol *func;
+ int idx;
+
+ immr = find_reloc_by_dest(elf, (void *)sec, offset+3);
+ disp = find_reloc_by_dest(elf, (void *)sec, offset+7);
+
+ if (!immr || strcmp(immr->sym->name, "pv_ops"))
+ break;
+
+ idx = (immr->addend + 8) / sizeof(void *);
+
+ func = disp->sym;
+ if (disp->sym->type == STT_SECTION)
+ func = find_symbol_by_offset(disp->sym->sec, disp->addend);
+ if (!func) {
+ WARN("no func for pv_ops[]");
+ return -1;
+ }
+
+ objtool_pv_add(file, idx, func);
+ }
+
+ break;
+
+ case 0xcf: /* iret */
+ /*
+ * Handle sync_core(), which has an IRET to self.
+ * All other IRET are in STT_NONE entry code.
+ */
+ sym = find_symbol_containing(sec, offset);
+ if (sym && sym->type == STT_FUNC) {
+ ADD_OP(op) {
+ /* add $40, %rsp */
+ op->src.type = OP_SRC_ADD;
+ op->src.reg = CFI_SP;
+ op->src.offset = 5*8;
+ op->dest.type = OP_DEST_REG;
+ op->dest.reg = CFI_SP;
+ }
+ break;
+ }
+
+ /* fallthrough */
+
+ case 0xca: /* retf */
+ case 0xcb: /* retf */
+ *type = INSN_CONTEXT_SWITCH;
+ break;
+
+ case 0xe0: /* loopne */
+ case 0xe1: /* loope */
+ case 0xe2: /* loop */
+ *type = INSN_JUMP_CONDITIONAL;
+ break;
+
+ case 0xe8:
+ *type = INSN_CALL;
+ /*
+ * For the impact on the stack, a CALL behaves like
+ * a PUSH of an immediate value (the return address).
+ */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_CONST;
+ op->dest.type = OP_DEST_PUSH;
+ }
+ break;
+
+ case 0xfc:
+ *type = INSN_CLD;
+ break;
+
+ case 0xfd:
+ *type = INSN_STD;
+ break;
+
+ case 0xff:
+ if (modrm_reg == 2 || modrm_reg == 3) {
+
+ *type = INSN_CALL_DYNAMIC;
+ if (has_notrack_prefix(&insn))
+ WARN("notrack prefix found at %s:0x%lx", sec->name, offset);
+
+ } else if (modrm_reg == 4) {
+
+ *type = INSN_JUMP_DYNAMIC;
+ if (has_notrack_prefix(&insn))
+ WARN("notrack prefix found at %s:0x%lx", sec->name, offset);
+
+ } else if (modrm_reg == 5) {
+
+ /* jmpf */
+ *type = INSN_CONTEXT_SWITCH;
+
+ } else if (modrm_reg == 6) {
+
+ /* push from mem */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_CONST;
+ op->dest.type = OP_DEST_PUSH;
+ }
+ }
+
+ break;
+
+ default:
+ break;
+ }
+
+ *immediate = insn.immediate.nbytes ? insn.immediate.value : 0;
+
+ return 0;
+}
+
+void arch_initial_func_cfi_state(struct cfi_init_state *state)
+{
+ int i;
+
+ for (i = 0; i < CFI_NUM_REGS; i++) {
+ state->regs[i].base = CFI_UNDEFINED;
+ state->regs[i].offset = 0;
+ }
+
+ /* initial CFA (call frame address) */
+ state->cfa.base = CFI_SP;
+ state->cfa.offset = 8;
+
+ /* initial RA (return address) */
+ state->regs[CFI_RA].base = CFI_CFA;
+ state->regs[CFI_RA].offset = -8;
+}
+
+const char *arch_nop_insn(int len)
+{
+ static const char nops[5][5] = {
+ { BYTES_NOP1 },
+ { BYTES_NOP2 },
+ { BYTES_NOP3 },
+ { BYTES_NOP4 },
+ { BYTES_NOP5 },
+ };
+
+ if (len < 1 || len > 5) {
+ WARN("invalid NOP size: %d\n", len);
+ return NULL;
+ }
+
+ return nops[len-1];
+}
+
+#define BYTE_RET 0xC3
+
+const char *arch_ret_insn(int len)
+{
+ static const char ret[5][5] = {
+ { BYTE_RET },
+ { BYTE_RET, 0xcc },
+ { BYTE_RET, 0xcc, BYTES_NOP1 },
+ { BYTE_RET, 0xcc, BYTES_NOP2 },
+ { BYTE_RET, 0xcc, BYTES_NOP3 },
+ };
+
+ if (len < 1 || len > 5) {
+ WARN("invalid RET size: %d\n", len);
+ return NULL;
+ }
+
+ return ret[len-1];
+}
+
+int arch_decode_hint_reg(u8 sp_reg, int *base)
+{
+ switch (sp_reg) {
+ case ORC_REG_UNDEFINED:
+ *base = CFI_UNDEFINED;
+ break;
+ case ORC_REG_SP:
+ *base = CFI_SP;
+ break;
+ case ORC_REG_BP:
+ *base = CFI_BP;
+ break;
+ case ORC_REG_SP_INDIRECT:
+ *base = CFI_SP_INDIRECT;
+ break;
+ case ORC_REG_R10:
+ *base = CFI_R10;
+ break;
+ case ORC_REG_R13:
+ *base = CFI_R13;
+ break;
+ case ORC_REG_DI:
+ *base = CFI_DI;
+ break;
+ case ORC_REG_DX:
+ *base = CFI_DX;
+ break;
+ default:
+ return -1;
+ }
+
+ return 0;
+}
+
+bool arch_is_retpoline(struct symbol *sym)
+{
+ return !strncmp(sym->name, "__x86_indirect_", 15);
+}
+
+bool arch_is_rethunk(struct symbol *sym)
+{
+ return !strcmp(sym->name, "__x86_return_thunk");
+}
+
+bool arch_is_embedded_insn(struct symbol *sym)
+{
+ return !strcmp(sym->name, "retbleed_return_thunk") ||
+ !strcmp(sym->name, "srso_safe_ret");
+}
diff --git a/tools/objtool/arch/x86/include/arch/cfi_regs.h b/tools/objtool/arch/x86/include/arch/cfi_regs.h
new file mode 100644
index 000000000..0579d22c4
--- /dev/null
+++ b/tools/objtool/arch/x86/include/arch/cfi_regs.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#ifndef _OBJTOOL_CFI_REGS_H
+#define _OBJTOOL_CFI_REGS_H
+
+#define CFI_AX 0
+#define CFI_CX 1
+#define CFI_DX 2
+#define CFI_BX 3
+#define CFI_SP 4
+#define CFI_BP 5
+#define CFI_SI 6
+#define CFI_DI 7
+#define CFI_R8 8
+#define CFI_R9 9
+#define CFI_R10 10
+#define CFI_R11 11
+#define CFI_R12 12
+#define CFI_R13 13
+#define CFI_R14 14
+#define CFI_R15 15
+#define CFI_RA 16
+#define CFI_NUM_REGS 17
+
+#endif /* _OBJTOOL_CFI_REGS_H */
diff --git a/tools/objtool/arch/x86/include/arch/elf.h b/tools/objtool/arch/x86/include/arch/elf.h
new file mode 100644
index 000000000..69cc4264b
--- /dev/null
+++ b/tools/objtool/arch/x86/include/arch/elf.h
@@ -0,0 +1,6 @@
+#ifndef _OBJTOOL_ARCH_ELF
+#define _OBJTOOL_ARCH_ELF
+
+#define R_NONE R_X86_64_NONE
+
+#endif /* _OBJTOOL_ARCH_ELF */
diff --git a/tools/objtool/arch/x86/include/arch/endianness.h b/tools/objtool/arch/x86/include/arch/endianness.h
new file mode 100644
index 000000000..7c362527d
--- /dev/null
+++ b/tools/objtool/arch/x86/include/arch/endianness.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _ARCH_ENDIANNESS_H
+#define _ARCH_ENDIANNESS_H
+
+#include <endian.h>
+
+#define __TARGET_BYTE_ORDER __LITTLE_ENDIAN
+
+#endif /* _ARCH_ENDIANNESS_H */
diff --git a/tools/objtool/arch/x86/include/arch/special.h b/tools/objtool/arch/x86/include/arch/special.h
new file mode 100644
index 000000000..f2918f789
--- /dev/null
+++ b/tools/objtool/arch/x86/include/arch/special.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _X86_ARCH_SPECIAL_H
+#define _X86_ARCH_SPECIAL_H
+
+#define EX_ENTRY_SIZE 12
+#define EX_ORIG_OFFSET 0
+#define EX_NEW_OFFSET 4
+
+#define JUMP_ENTRY_SIZE 16
+#define JUMP_ORIG_OFFSET 0
+#define JUMP_NEW_OFFSET 4
+#define JUMP_KEY_OFFSET 8
+
+#define ALT_ENTRY_SIZE 12
+#define ALT_ORIG_OFFSET 0
+#define ALT_NEW_OFFSET 4
+#define ALT_FEATURE_OFFSET 8
+#define ALT_ORIG_LEN_OFFSET 10
+#define ALT_NEW_LEN_OFFSET 11
+
+#endif /* _X86_ARCH_SPECIAL_H */
diff --git a/tools/objtool/arch/x86/special.c b/tools/objtool/arch/x86/special.c
new file mode 100644
index 000000000..7c97b7391
--- /dev/null
+++ b/tools/objtool/arch/x86/special.c
@@ -0,0 +1,145 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <string.h>
+
+#include <objtool/special.h>
+#include <objtool/builtin.h>
+
+#define X86_FEATURE_POPCNT (4 * 32 + 23)
+#define X86_FEATURE_SMAP (9 * 32 + 20)
+
+void arch_handle_alternative(unsigned short feature, struct special_alt *alt)
+{
+ switch (feature) {
+ case X86_FEATURE_SMAP:
+ /*
+ * If UACCESS validation is enabled; force that alternative;
+ * otherwise force it the other way.
+ *
+ * What we want to avoid is having both the original and the
+ * alternative code flow at the same time, in that case we can
+ * find paths that see the STAC but take the NOP instead of
+ * CLAC and the other way around.
+ */
+ if (opts.uaccess)
+ alt->skip_orig = true;
+ else
+ alt->skip_alt = true;
+ break;
+ case X86_FEATURE_POPCNT:
+ /*
+ * It has been requested that we don't validate the !POPCNT
+ * feature path which is a "very very small percentage of
+ * machines".
+ */
+ alt->skip_orig = true;
+ break;
+ default:
+ break;
+ }
+}
+
+bool arch_support_alt_relocation(struct special_alt *special_alt,
+ struct instruction *insn,
+ struct reloc *reloc)
+{
+ /*
+ * The x86 alternatives code adjusts the offsets only when it
+ * encounters a branch instruction at the very beginning of the
+ * replacement group.
+ */
+ return insn->offset == special_alt->new_off &&
+ (insn->type == INSN_CALL || is_jump(insn));
+}
+
+/*
+ * There are 3 basic jump table patterns:
+ *
+ * 1. jmpq *[rodata addr](,%reg,8)
+ *
+ * This is the most common case by far. It jumps to an address in a simple
+ * jump table which is stored in .rodata.
+ *
+ * 2. jmpq *[rodata addr](%rip)
+ *
+ * This is caused by a rare GCC quirk, currently only seen in three driver
+ * functions in the kernel, only with certain obscure non-distro configs.
+ *
+ * As part of an optimization, GCC makes a copy of an existing switch jump
+ * table, modifies it, and then hard-codes the jump (albeit with an indirect
+ * jump) to use a single entry in the table. The rest of the jump table and
+ * some of its jump targets remain as dead code.
+ *
+ * In such a case we can just crudely ignore all unreachable instruction
+ * warnings for the entire object file. Ideally we would just ignore them
+ * for the function, but that would require redesigning the code quite a
+ * bit. And honestly that's just not worth doing: unreachable instruction
+ * warnings are of questionable value anyway, and this is such a rare issue.
+ *
+ * 3. mov [rodata addr],%reg1
+ * ... some instructions ...
+ * jmpq *(%reg1,%reg2,8)
+ *
+ * This is a fairly uncommon pattern which is new for GCC 6. As of this
+ * writing, there are 11 occurrences of it in the allmodconfig kernel.
+ *
+ * As of GCC 7 there are quite a few more of these and the 'in between' code
+ * is significant. Esp. with KASAN enabled some of the code between the mov
+ * and jmpq uses .rodata itself, which can confuse things.
+ *
+ * TODO: Once we have DWARF CFI and smarter instruction decoding logic,
+ * ensure the same register is used in the mov and jump instructions.
+ *
+ * NOTE: RETPOLINE made it harder still to decode dynamic jumps.
+ */
+struct reloc *arch_find_switch_table(struct objtool_file *file,
+ struct instruction *insn)
+{
+ struct reloc *text_reloc, *rodata_reloc;
+ struct section *table_sec;
+ unsigned long table_offset;
+
+ /* look for a relocation which references .rodata */
+ text_reloc = find_reloc_by_dest_range(file->elf, insn->sec,
+ insn->offset, insn->len);
+ if (!text_reloc || text_reloc->sym->type != STT_SECTION ||
+ !text_reloc->sym->sec->rodata)
+ return NULL;
+
+ table_offset = text_reloc->addend;
+ table_sec = text_reloc->sym->sec;
+
+ if (text_reloc->type == R_X86_64_PC32)
+ table_offset += 4;
+
+ /*
+ * Make sure the .rodata address isn't associated with a
+ * symbol. GCC jump tables are anonymous data.
+ *
+ * Also support C jump tables which are in the same format as
+ * switch jump tables. For objtool to recognize them, they
+ * need to be placed in the C_JUMP_TABLE_SECTION section. They
+ * have symbols associated with them.
+ */
+ if (find_symbol_containing(table_sec, table_offset) &&
+ strcmp(table_sec->name, C_JUMP_TABLE_SECTION))
+ return NULL;
+
+ /*
+ * Each table entry has a rela associated with it. The rela
+ * should reference text in the same function as the original
+ * instruction.
+ */
+ rodata_reloc = find_reloc_by_dest(file->elf, table_sec, table_offset);
+ if (!rodata_reloc)
+ return NULL;
+
+ /*
+ * Use of RIP-relative switch jumps is quite rare, and
+ * indicates a rare GCC quirk/bug which can leave dead
+ * code behind.
+ */
+ if (text_reloc->type == R_X86_64_PC32)
+ file->ignore_unreachables = true;
+
+ return rodata_reloc;
+}
diff --git a/tools/objtool/builtin-check.c b/tools/objtool/builtin-check.c
new file mode 100644
index 000000000..24fbe803a
--- /dev/null
+++ b/tools/objtool/builtin-check.c
@@ -0,0 +1,212 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
+ */
+
+#include <subcmd/parse-options.h>
+#include <string.h>
+#include <stdlib.h>
+#include <objtool/builtin.h>
+#include <objtool/objtool.h>
+
+#define ERROR(format, ...) \
+ fprintf(stderr, \
+ "error: objtool: " format "\n", \
+ ##__VA_ARGS__)
+
+struct opts opts;
+
+static const char * const check_usage[] = {
+ "objtool <actions> [<options>] file.o",
+ NULL,
+};
+
+static const char * const env_usage[] = {
+ "OBJTOOL_ARGS=\"<options>\"",
+ NULL,
+};
+
+static int parse_dump(const struct option *opt, const char *str, int unset)
+{
+ if (!str || !strcmp(str, "orc")) {
+ opts.dump_orc = true;
+ return 0;
+ }
+
+ return -1;
+}
+
+static int parse_hacks(const struct option *opt, const char *str, int unset)
+{
+ bool found = false;
+
+ /*
+ * Use strstr() as a lazy method of checking for comma-separated
+ * options.
+ *
+ * No string provided == enable all options.
+ */
+
+ if (!str || strstr(str, "jump_label")) {
+ opts.hack_jump_label = true;
+ found = true;
+ }
+
+ if (!str || strstr(str, "noinstr")) {
+ opts.hack_noinstr = true;
+ found = true;
+ }
+
+ return found ? 0 : -1;
+}
+
+const struct option check_options[] = {
+ OPT_GROUP("Actions:"),
+ OPT_CALLBACK_OPTARG('h', "hacks", NULL, NULL, "jump_label,noinstr", "patch toolchain bugs/limitations", parse_hacks),
+ OPT_BOOLEAN('i', "ibt", &opts.ibt, "validate and annotate IBT"),
+ OPT_BOOLEAN('m', "mcount", &opts.mcount, "annotate mcount/fentry calls for ftrace"),
+ OPT_BOOLEAN('n', "noinstr", &opts.noinstr, "validate noinstr rules"),
+ OPT_BOOLEAN('o', "orc", &opts.orc, "generate ORC metadata"),
+ OPT_BOOLEAN('r', "retpoline", &opts.retpoline, "validate and annotate retpoline usage"),
+ OPT_BOOLEAN(0, "rethunk", &opts.rethunk, "validate and annotate rethunk usage"),
+ OPT_BOOLEAN(0, "unret", &opts.unret, "validate entry unret placement"),
+ OPT_BOOLEAN('l', "sls", &opts.sls, "validate straight-line-speculation mitigations"),
+ OPT_BOOLEAN('s', "stackval", &opts.stackval, "validate frame pointer rules"),
+ OPT_BOOLEAN('t', "static-call", &opts.static_call, "annotate static calls"),
+ OPT_BOOLEAN('u', "uaccess", &opts.uaccess, "validate uaccess rules for SMAP"),
+ OPT_CALLBACK_OPTARG(0, "dump", NULL, NULL, "orc", "dump metadata", parse_dump),
+
+ OPT_GROUP("Options:"),
+ OPT_BOOLEAN(0, "backtrace", &opts.backtrace, "unwind on error"),
+ OPT_BOOLEAN(0, "backup", &opts.backup, "create .orig files before modification"),
+ OPT_BOOLEAN(0, "dry-run", &opts.dryrun, "don't write modifications"),
+ OPT_BOOLEAN(0, "link", &opts.link, "object is a linked object"),
+ OPT_BOOLEAN(0, "module", &opts.module, "object is part of a kernel module"),
+ OPT_BOOLEAN(0, "no-unreachable", &opts.no_unreachable, "skip 'unreachable instruction' warnings"),
+ OPT_BOOLEAN(0, "sec-address", &opts.sec_address, "print section addresses in warnings"),
+ OPT_BOOLEAN(0, "stats", &opts.stats, "print statistics"),
+
+ OPT_END(),
+};
+
+int cmd_parse_options(int argc, const char **argv, const char * const usage[])
+{
+ const char *envv[16] = { };
+ char *env;
+ int envc;
+
+ env = getenv("OBJTOOL_ARGS");
+ if (env) {
+ envv[0] = "OBJTOOL_ARGS";
+ for (envc = 1; envc < ARRAY_SIZE(envv); ) {
+ envv[envc++] = env;
+ env = strchr(env, ' ');
+ if (!env)
+ break;
+ *env = '\0';
+ env++;
+ }
+
+ parse_options(envc, envv, check_options, env_usage, 0);
+ }
+
+ argc = parse_options(argc, argv, check_options, usage, 0);
+ if (argc != 1)
+ usage_with_options(usage, check_options);
+ return argc;
+}
+
+static bool opts_valid(void)
+{
+ if (opts.hack_jump_label ||
+ opts.hack_noinstr ||
+ opts.ibt ||
+ opts.mcount ||
+ opts.noinstr ||
+ opts.orc ||
+ opts.retpoline ||
+ opts.rethunk ||
+ opts.sls ||
+ opts.stackval ||
+ opts.static_call ||
+ opts.uaccess) {
+ if (opts.dump_orc) {
+ ERROR("--dump can't be combined with other options");
+ return false;
+ }
+
+ return true;
+ }
+
+ if (opts.unret && !opts.rethunk) {
+ ERROR("--unret requires --rethunk");
+ return false;
+ }
+
+ if (opts.dump_orc)
+ return true;
+
+ ERROR("At least one command required");
+ return false;
+}
+
+static bool link_opts_valid(struct objtool_file *file)
+{
+ if (opts.link)
+ return true;
+
+ if (has_multiple_files(file->elf)) {
+ ERROR("Linked object detected, forcing --link");
+ opts.link = true;
+ return true;
+ }
+
+ if (opts.noinstr) {
+ ERROR("--noinstr requires --link");
+ return false;
+ }
+
+ if (opts.ibt) {
+ ERROR("--ibt requires --link");
+ return false;
+ }
+
+ if (opts.unret) {
+ ERROR("--unret requires --link");
+ return false;
+ }
+
+ return true;
+}
+
+int objtool_run(int argc, const char **argv)
+{
+ const char *objname;
+ struct objtool_file *file;
+ int ret;
+
+ argc = cmd_parse_options(argc, argv, check_usage);
+ objname = argv[0];
+
+ if (!opts_valid())
+ return 1;
+
+ if (opts.dump_orc)
+ return orc_dump(objname);
+
+ file = objtool_open_read(objname);
+ if (!file)
+ return 1;
+
+ if (!link_opts_valid(file))
+ return 1;
+
+ ret = check(file);
+ if (ret)
+ return ret;
+
+ if (file->elf->changed)
+ return elf_write(file->elf);
+
+ return 0;
+}
diff --git a/tools/objtool/check.c b/tools/objtool/check.c
new file mode 100644
index 000000000..cb363b507
--- /dev/null
+++ b/tools/objtool/check.c
@@ -0,0 +1,4371 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
+ */
+
+#include <string.h>
+#include <stdlib.h>
+#include <inttypes.h>
+#include <sys/mman.h>
+
+#include <arch/elf.h>
+#include <objtool/builtin.h>
+#include <objtool/cfi.h>
+#include <objtool/arch.h>
+#include <objtool/check.h>
+#include <objtool/special.h>
+#include <objtool/warn.h>
+#include <objtool/endianness.h>
+
+#include <linux/objtool.h>
+#include <linux/hashtable.h>
+#include <linux/kernel.h>
+#include <linux/static_call_types.h>
+
+struct alternative {
+ struct list_head list;
+ struct instruction *insn;
+ bool skip_orig;
+};
+
+static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
+
+static struct cfi_init_state initial_func_cfi;
+static struct cfi_state init_cfi;
+static struct cfi_state func_cfi;
+
+struct instruction *find_insn(struct objtool_file *file,
+ struct section *sec, unsigned long offset)
+{
+ struct instruction *insn;
+
+ hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
+ if (insn->sec == sec && insn->offset == offset)
+ return insn;
+ }
+
+ return NULL;
+}
+
+static struct instruction *next_insn_same_sec(struct objtool_file *file,
+ struct instruction *insn)
+{
+ struct instruction *next = list_next_entry(insn, list);
+
+ if (!next || &next->list == &file->insn_list || next->sec != insn->sec)
+ return NULL;
+
+ return next;
+}
+
+static struct instruction *next_insn_same_func(struct objtool_file *file,
+ struct instruction *insn)
+{
+ struct instruction *next = list_next_entry(insn, list);
+ struct symbol *func = insn->func;
+
+ if (!func)
+ return NULL;
+
+ if (&next->list != &file->insn_list && next->func == func)
+ return next;
+
+ /* Check if we're already in the subfunction: */
+ if (func == func->cfunc)
+ return NULL;
+
+ /* Move to the subfunction: */
+ return find_insn(file, func->cfunc->sec, func->cfunc->offset);
+}
+
+static struct instruction *prev_insn_same_sym(struct objtool_file *file,
+ struct instruction *insn)
+{
+ struct instruction *prev = list_prev_entry(insn, list);
+
+ if (&prev->list != &file->insn_list && prev->func == insn->func)
+ return prev;
+
+ return NULL;
+}
+
+#define func_for_each_insn(file, func, insn) \
+ for (insn = find_insn(file, func->sec, func->offset); \
+ insn; \
+ insn = next_insn_same_func(file, insn))
+
+#define sym_for_each_insn(file, sym, insn) \
+ for (insn = find_insn(file, sym->sec, sym->offset); \
+ insn && &insn->list != &file->insn_list && \
+ insn->sec == sym->sec && \
+ insn->offset < sym->offset + sym->len; \
+ insn = list_next_entry(insn, list))
+
+#define sym_for_each_insn_continue_reverse(file, sym, insn) \
+ for (insn = list_prev_entry(insn, list); \
+ &insn->list != &file->insn_list && \
+ insn->sec == sym->sec && insn->offset >= sym->offset; \
+ insn = list_prev_entry(insn, list))
+
+#define sec_for_each_insn_from(file, insn) \
+ for (; insn; insn = next_insn_same_sec(file, insn))
+
+#define sec_for_each_insn_continue(file, insn) \
+ for (insn = next_insn_same_sec(file, insn); insn; \
+ insn = next_insn_same_sec(file, insn))
+
+static bool is_jump_table_jump(struct instruction *insn)
+{
+ struct alt_group *alt_group = insn->alt_group;
+
+ if (insn->jump_table)
+ return true;
+
+ /* Retpoline alternative for a jump table? */
+ return alt_group && alt_group->orig_group &&
+ alt_group->orig_group->first_insn->jump_table;
+}
+
+static bool is_sibling_call(struct instruction *insn)
+{
+ /*
+ * Assume only ELF functions can make sibling calls. This ensures
+ * sibling call detection consistency between vmlinux.o and individual
+ * objects.
+ */
+ if (!insn->func)
+ return false;
+
+ /* An indirect jump is either a sibling call or a jump to a table. */
+ if (insn->type == INSN_JUMP_DYNAMIC)
+ return !is_jump_table_jump(insn);
+
+ /* add_jump_destinations() sets insn->call_dest for sibling calls. */
+ return (is_static_jump(insn) && insn->call_dest);
+}
+
+/*
+ * This checks to see if the given function is a "noreturn" function.
+ *
+ * For global functions which are outside the scope of this object file, we
+ * have to keep a manual list of them.
+ *
+ * For local functions, we have to detect them manually by simply looking for
+ * the lack of a return instruction.
+ */
+static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
+ int recursion)
+{
+ int i;
+ struct instruction *insn;
+ bool empty = true;
+
+ /*
+ * Unfortunately these have to be hard coded because the noreturn
+ * attribute isn't provided in ELF data. Keep 'em sorted.
+ */
+ static const char * const global_noreturns[] = {
+ "__invalid_creds",
+ "__module_put_and_kthread_exit",
+ "__reiserfs_panic",
+ "__stack_chk_fail",
+ "__ubsan_handle_builtin_unreachable",
+ "cpu_bringup_and_idle",
+ "cpu_startup_entry",
+ "do_exit",
+ "do_group_exit",
+ "do_task_dead",
+ "ex_handler_msr_mce",
+ "fortify_panic",
+ "kthread_complete_and_exit",
+ "kthread_exit",
+ "kunit_try_catch_throw",
+ "lbug_with_loc",
+ "machine_real_restart",
+ "make_task_dead",
+ "panic",
+ "rewind_stack_and_make_dead",
+ "sev_es_terminate",
+ "snp_abort",
+ "stop_this_cpu",
+ "usercopy_abort",
+ "xen_start_kernel",
+ };
+
+ if (!func)
+ return false;
+
+ if (func->bind == STB_WEAK)
+ return false;
+
+ if (func->bind == STB_GLOBAL)
+ for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
+ if (!strcmp(func->name, global_noreturns[i]))
+ return true;
+
+ if (!func->len)
+ return false;
+
+ insn = find_insn(file, func->sec, func->offset);
+ if (!insn || !insn->func)
+ return false;
+
+ func_for_each_insn(file, func, insn) {
+ empty = false;
+
+ if (insn->type == INSN_RETURN)
+ return false;
+ }
+
+ if (empty)
+ return false;
+
+ /*
+ * A function can have a sibling call instead of a return. In that
+ * case, the function's dead-end status depends on whether the target
+ * of the sibling call returns.
+ */
+ func_for_each_insn(file, func, insn) {
+ if (is_sibling_call(insn)) {
+ struct instruction *dest = insn->jump_dest;
+
+ if (!dest)
+ /* sibling call to another file */
+ return false;
+
+ /* local sibling call */
+ if (recursion == 5) {
+ /*
+ * Infinite recursion: two functions have
+ * sibling calls to each other. This is a very
+ * rare case. It means they aren't dead ends.
+ */
+ return false;
+ }
+
+ return __dead_end_function(file, dest->func, recursion+1);
+ }
+ }
+
+ return true;
+}
+
+static bool dead_end_function(struct objtool_file *file, struct symbol *func)
+{
+ return __dead_end_function(file, func, 0);
+}
+
+static void init_cfi_state(struct cfi_state *cfi)
+{
+ int i;
+
+ for (i = 0; i < CFI_NUM_REGS; i++) {
+ cfi->regs[i].base = CFI_UNDEFINED;
+ cfi->vals[i].base = CFI_UNDEFINED;
+ }
+ cfi->cfa.base = CFI_UNDEFINED;
+ cfi->drap_reg = CFI_UNDEFINED;
+ cfi->drap_offset = -1;
+}
+
+static void init_insn_state(struct objtool_file *file, struct insn_state *state,
+ struct section *sec)
+{
+ memset(state, 0, sizeof(*state));
+ init_cfi_state(&state->cfi);
+
+ /*
+ * We need the full vmlinux for noinstr validation, otherwise we can
+ * not correctly determine insn->call_dest->sec (external symbols do
+ * not have a section).
+ */
+ if (opts.link && opts.noinstr && sec)
+ state->noinstr = sec->noinstr;
+}
+
+static struct cfi_state *cfi_alloc(void)
+{
+ struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
+ if (!cfi) {
+ WARN("calloc failed");
+ exit(1);
+ }
+ nr_cfi++;
+ return cfi;
+}
+
+static int cfi_bits;
+static struct hlist_head *cfi_hash;
+
+static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
+{
+ return memcmp((void *)cfi1 + sizeof(cfi1->hash),
+ (void *)cfi2 + sizeof(cfi2->hash),
+ sizeof(struct cfi_state) - sizeof(struct hlist_node));
+}
+
+static inline u32 cfi_key(struct cfi_state *cfi)
+{
+ return jhash((void *)cfi + sizeof(cfi->hash),
+ sizeof(*cfi) - sizeof(cfi->hash), 0);
+}
+
+static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
+{
+ struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
+ struct cfi_state *obj;
+
+ hlist_for_each_entry(obj, head, hash) {
+ if (!cficmp(cfi, obj)) {
+ nr_cfi_cache++;
+ return obj;
+ }
+ }
+
+ obj = cfi_alloc();
+ *obj = *cfi;
+ hlist_add_head(&obj->hash, head);
+
+ return obj;
+}
+
+static void cfi_hash_add(struct cfi_state *cfi)
+{
+ struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
+
+ hlist_add_head(&cfi->hash, head);
+}
+
+static void *cfi_hash_alloc(unsigned long size)
+{
+ cfi_bits = max(10, ilog2(size));
+ cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
+ PROT_READ|PROT_WRITE,
+ MAP_PRIVATE|MAP_ANON, -1, 0);
+ if (cfi_hash == (void *)-1L) {
+ WARN("mmap fail cfi_hash");
+ cfi_hash = NULL;
+ } else if (opts.stats) {
+ printf("cfi_bits: %d\n", cfi_bits);
+ }
+
+ return cfi_hash;
+}
+
+static unsigned long nr_insns;
+static unsigned long nr_insns_visited;
+
+/*
+ * Call the arch-specific instruction decoder for all the instructions and add
+ * them to the global instruction list.
+ */
+static int decode_instructions(struct objtool_file *file)
+{
+ struct section *sec;
+ struct symbol *func;
+ unsigned long offset;
+ struct instruction *insn;
+ int ret;
+
+ for_each_sec(file, sec) {
+
+ if (!(sec->sh.sh_flags & SHF_EXECINSTR))
+ continue;
+
+ if (strcmp(sec->name, ".altinstr_replacement") &&
+ strcmp(sec->name, ".altinstr_aux") &&
+ strncmp(sec->name, ".discard.", 9))
+ sec->text = true;
+
+ if (!strcmp(sec->name, ".noinstr.text") ||
+ !strcmp(sec->name, ".entry.text") ||
+ !strncmp(sec->name, ".text..__x86.", 13))
+ sec->noinstr = true;
+
+ for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
+ insn = malloc(sizeof(*insn));
+ if (!insn) {
+ WARN("malloc failed");
+ return -1;
+ }
+ memset(insn, 0, sizeof(*insn));
+ INIT_LIST_HEAD(&insn->alts);
+ INIT_LIST_HEAD(&insn->stack_ops);
+ INIT_LIST_HEAD(&insn->call_node);
+
+ insn->sec = sec;
+ insn->offset = offset;
+
+ ret = arch_decode_instruction(file, sec, offset,
+ sec->sh.sh_size - offset,
+ &insn->len, &insn->type,
+ &insn->immediate,
+ &insn->stack_ops);
+ if (ret)
+ goto err;
+
+ /*
+ * By default, "ud2" is a dead end unless otherwise
+ * annotated, because GCC 7 inserts it for certain
+ * divide-by-zero cases.
+ */
+ if (insn->type == INSN_BUG)
+ insn->dead_end = true;
+
+ hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
+ list_add_tail(&insn->list, &file->insn_list);
+ nr_insns++;
+ }
+
+ list_for_each_entry(func, &sec->symbol_list, list) {
+ if (func->type != STT_FUNC || func->alias != func)
+ continue;
+
+ if (!find_insn(file, sec, func->offset)) {
+ WARN("%s(): can't find starting instruction",
+ func->name);
+ return -1;
+ }
+
+ sym_for_each_insn(file, func, insn) {
+ insn->func = func;
+ if (insn->type == INSN_ENDBR && list_empty(&insn->call_node)) {
+ if (insn->offset == insn->func->offset) {
+ list_add_tail(&insn->call_node, &file->endbr_list);
+ file->nr_endbr++;
+ } else {
+ file->nr_endbr_int++;
+ }
+ }
+ }
+ }
+ }
+
+ if (opts.stats)
+ printf("nr_insns: %lu\n", nr_insns);
+
+ return 0;
+
+err:
+ free(insn);
+ return ret;
+}
+
+/*
+ * Read the pv_ops[] .data table to find the static initialized values.
+ */
+static int add_pv_ops(struct objtool_file *file, const char *symname)
+{
+ struct symbol *sym, *func;
+ unsigned long off, end;
+ struct reloc *rel;
+ int idx;
+
+ sym = find_symbol_by_name(file->elf, symname);
+ if (!sym)
+ return 0;
+
+ off = sym->offset;
+ end = off + sym->len;
+ for (;;) {
+ rel = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
+ if (!rel)
+ break;
+
+ func = rel->sym;
+ if (func->type == STT_SECTION)
+ func = find_symbol_by_offset(rel->sym->sec, rel->addend);
+
+ idx = (rel->offset - sym->offset) / sizeof(unsigned long);
+
+ objtool_pv_add(file, idx, func);
+
+ off = rel->offset + 1;
+ if (off > end)
+ break;
+ }
+
+ return 0;
+}
+
+/*
+ * Allocate and initialize file->pv_ops[].
+ */
+static int init_pv_ops(struct objtool_file *file)
+{
+ static const char *pv_ops_tables[] = {
+ "pv_ops",
+ "xen_cpu_ops",
+ "xen_irq_ops",
+ "xen_mmu_ops",
+ NULL,
+ };
+ const char *pv_ops;
+ struct symbol *sym;
+ int idx, nr;
+
+ if (!opts.noinstr)
+ return 0;
+
+ file->pv_ops = NULL;
+
+ sym = find_symbol_by_name(file->elf, "pv_ops");
+ if (!sym)
+ return 0;
+
+ nr = sym->len / sizeof(unsigned long);
+ file->pv_ops = calloc(sizeof(struct pv_state), nr);
+ if (!file->pv_ops)
+ return -1;
+
+ for (idx = 0; idx < nr; idx++)
+ INIT_LIST_HEAD(&file->pv_ops[idx].targets);
+
+ for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
+ add_pv_ops(file, pv_ops);
+
+ return 0;
+}
+
+static struct instruction *find_last_insn(struct objtool_file *file,
+ struct section *sec)
+{
+ struct instruction *insn = NULL;
+ unsigned int offset;
+ unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
+
+ for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
+ insn = find_insn(file, sec, offset);
+
+ return insn;
+}
+
+/*
+ * Mark "ud2" instructions and manually annotated dead ends.
+ */
+static int add_dead_ends(struct objtool_file *file)
+{
+ struct section *sec;
+ struct reloc *reloc;
+ struct instruction *insn;
+
+ /*
+ * Check for manually annotated dead ends.
+ */
+ sec = find_section_by_name(file->elf, ".rela.discard.unreachable");
+ if (!sec)
+ goto reachable;
+
+ list_for_each_entry(reloc, &sec->reloc_list, list) {
+ if (reloc->sym->type != STT_SECTION) {
+ WARN("unexpected relocation symbol type in %s", sec->name);
+ return -1;
+ }
+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
+ if (insn)
+ insn = list_prev_entry(insn, list);
+ else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
+ insn = find_last_insn(file, reloc->sym->sec);
+ if (!insn) {
+ WARN("can't find unreachable insn at %s+0x%" PRIx64,
+ reloc->sym->sec->name, reloc->addend);
+ return -1;
+ }
+ } else {
+ WARN("can't find unreachable insn at %s+0x%" PRIx64,
+ reloc->sym->sec->name, reloc->addend);
+ return -1;
+ }
+
+ insn->dead_end = true;
+ }
+
+reachable:
+ /*
+ * These manually annotated reachable checks are needed for GCC 4.4,
+ * where the Linux unreachable() macro isn't supported. In that case
+ * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
+ * not a dead end.
+ */
+ sec = find_section_by_name(file->elf, ".rela.discard.reachable");
+ if (!sec)
+ return 0;
+
+ list_for_each_entry(reloc, &sec->reloc_list, list) {
+ if (reloc->sym->type != STT_SECTION) {
+ WARN("unexpected relocation symbol type in %s", sec->name);
+ return -1;
+ }
+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
+ if (insn)
+ insn = list_prev_entry(insn, list);
+ else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
+ insn = find_last_insn(file, reloc->sym->sec);
+ if (!insn) {
+ WARN("can't find reachable insn at %s+0x%" PRIx64,
+ reloc->sym->sec->name, reloc->addend);
+ return -1;
+ }
+ } else {
+ WARN("can't find reachable insn at %s+0x%" PRIx64,
+ reloc->sym->sec->name, reloc->addend);
+ return -1;
+ }
+
+ insn->dead_end = false;
+ }
+
+ return 0;
+}
+
+static int create_static_call_sections(struct objtool_file *file)
+{
+ struct section *sec;
+ struct static_call_site *site;
+ struct instruction *insn;
+ struct symbol *key_sym;
+ char *key_name, *tmp;
+ int idx;
+
+ sec = find_section_by_name(file->elf, ".static_call_sites");
+ if (sec) {
+ INIT_LIST_HEAD(&file->static_call_list);
+ WARN("file already has .static_call_sites section, skipping");
+ return 0;
+ }
+
+ if (list_empty(&file->static_call_list))
+ return 0;
+
+ idx = 0;
+ list_for_each_entry(insn, &file->static_call_list, call_node)
+ idx++;
+
+ sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
+ sizeof(struct static_call_site), idx);
+ if (!sec)
+ return -1;
+
+ idx = 0;
+ list_for_each_entry(insn, &file->static_call_list, call_node) {
+
+ site = (struct static_call_site *)sec->data->d_buf + idx;
+ memset(site, 0, sizeof(struct static_call_site));
+
+ /* populate reloc for 'addr' */
+ if (elf_add_reloc_to_insn(file->elf, sec,
+ idx * sizeof(struct static_call_site),
+ R_X86_64_PC32,
+ insn->sec, insn->offset))
+ return -1;
+
+ /* find key symbol */
+ key_name = strdup(insn->call_dest->name);
+ if (!key_name) {
+ perror("strdup");
+ return -1;
+ }
+ if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
+ STATIC_CALL_TRAMP_PREFIX_LEN)) {
+ WARN("static_call: trampoline name malformed: %s", key_name);
+ free(key_name);
+ return -1;
+ }
+ tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
+ memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
+
+ key_sym = find_symbol_by_name(file->elf, tmp);
+ if (!key_sym) {
+ if (!opts.module) {
+ WARN("static_call: can't find static_call_key symbol: %s", tmp);
+ free(key_name);
+ return -1;
+ }
+
+ /*
+ * For modules(), the key might not be exported, which
+ * means the module can make static calls but isn't
+ * allowed to change them.
+ *
+ * In that case we temporarily set the key to be the
+ * trampoline address. This is fixed up in
+ * static_call_add_module().
+ */
+ key_sym = insn->call_dest;
+ }
+ free(key_name);
+
+ /* populate reloc for 'key' */
+ if (elf_add_reloc(file->elf, sec,
+ idx * sizeof(struct static_call_site) + 4,
+ R_X86_64_PC32, key_sym,
+ is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
+ return -1;
+
+ idx++;
+ }
+
+ return 0;
+}
+
+static int create_retpoline_sites_sections(struct objtool_file *file)
+{
+ struct instruction *insn;
+ struct section *sec;
+ int idx;
+
+ sec = find_section_by_name(file->elf, ".retpoline_sites");
+ if (sec) {
+ WARN("file already has .retpoline_sites, skipping");
+ return 0;
+ }
+
+ idx = 0;
+ list_for_each_entry(insn, &file->retpoline_call_list, call_node)
+ idx++;
+
+ if (!idx)
+ return 0;
+
+ sec = elf_create_section(file->elf, ".retpoline_sites", 0,
+ sizeof(int), idx);
+ if (!sec) {
+ WARN("elf_create_section: .retpoline_sites");
+ return -1;
+ }
+
+ idx = 0;
+ list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
+
+ int *site = (int *)sec->data->d_buf + idx;
+ *site = 0;
+
+ if (elf_add_reloc_to_insn(file->elf, sec,
+ idx * sizeof(int),
+ R_X86_64_PC32,
+ insn->sec, insn->offset)) {
+ WARN("elf_add_reloc_to_insn: .retpoline_sites");
+ return -1;
+ }
+
+ idx++;
+ }
+
+ return 0;
+}
+
+static int create_return_sites_sections(struct objtool_file *file)
+{
+ struct instruction *insn;
+ struct section *sec;
+ int idx;
+
+ sec = find_section_by_name(file->elf, ".return_sites");
+ if (sec) {
+ WARN("file already has .return_sites, skipping");
+ return 0;
+ }
+
+ idx = 0;
+ list_for_each_entry(insn, &file->return_thunk_list, call_node)
+ idx++;
+
+ if (!idx)
+ return 0;
+
+ sec = elf_create_section(file->elf, ".return_sites", 0,
+ sizeof(int), idx);
+ if (!sec) {
+ WARN("elf_create_section: .return_sites");
+ return -1;
+ }
+
+ idx = 0;
+ list_for_each_entry(insn, &file->return_thunk_list, call_node) {
+
+ int *site = (int *)sec->data->d_buf + idx;
+ *site = 0;
+
+ if (elf_add_reloc_to_insn(file->elf, sec,
+ idx * sizeof(int),
+ R_X86_64_PC32,
+ insn->sec, insn->offset)) {
+ WARN("elf_add_reloc_to_insn: .return_sites");
+ return -1;
+ }
+
+ idx++;
+ }
+
+ return 0;
+}
+
+static int create_ibt_endbr_seal_sections(struct objtool_file *file)
+{
+ struct instruction *insn;
+ struct section *sec;
+ int idx;
+
+ sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
+ if (sec) {
+ WARN("file already has .ibt_endbr_seal, skipping");
+ return 0;
+ }
+
+ idx = 0;
+ list_for_each_entry(insn, &file->endbr_list, call_node)
+ idx++;
+
+ if (opts.stats) {
+ printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
+ printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
+ printf("ibt: superfluous ENDBR: %d\n", idx);
+ }
+
+ if (!idx)
+ return 0;
+
+ sec = elf_create_section(file->elf, ".ibt_endbr_seal", 0,
+ sizeof(int), idx);
+ if (!sec) {
+ WARN("elf_create_section: .ibt_endbr_seal");
+ return -1;
+ }
+
+ idx = 0;
+ list_for_each_entry(insn, &file->endbr_list, call_node) {
+
+ int *site = (int *)sec->data->d_buf + idx;
+ *site = 0;
+
+ if (elf_add_reloc_to_insn(file->elf, sec,
+ idx * sizeof(int),
+ R_X86_64_PC32,
+ insn->sec, insn->offset)) {
+ WARN("elf_add_reloc_to_insn: .ibt_endbr_seal");
+ return -1;
+ }
+
+ idx++;
+ }
+
+ return 0;
+}
+
+static int create_mcount_loc_sections(struct objtool_file *file)
+{
+ struct section *sec;
+ unsigned long *loc;
+ struct instruction *insn;
+ int idx;
+
+ sec = find_section_by_name(file->elf, "__mcount_loc");
+ if (sec) {
+ INIT_LIST_HEAD(&file->mcount_loc_list);
+ WARN("file already has __mcount_loc section, skipping");
+ return 0;
+ }
+
+ if (list_empty(&file->mcount_loc_list))
+ return 0;
+
+ idx = 0;
+ list_for_each_entry(insn, &file->mcount_loc_list, call_node)
+ idx++;
+
+ sec = elf_create_section(file->elf, "__mcount_loc", 0, sizeof(unsigned long), idx);
+ if (!sec)
+ return -1;
+
+ idx = 0;
+ list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
+
+ loc = (unsigned long *)sec->data->d_buf + idx;
+ memset(loc, 0, sizeof(unsigned long));
+
+ if (elf_add_reloc_to_insn(file->elf, sec,
+ idx * sizeof(unsigned long),
+ R_X86_64_64,
+ insn->sec, insn->offset))
+ return -1;
+
+ idx++;
+ }
+
+ return 0;
+}
+
+/*
+ * Warnings shouldn't be reported for ignored functions.
+ */
+static void add_ignores(struct objtool_file *file)
+{
+ struct instruction *insn;
+ struct section *sec;
+ struct symbol *func;
+ struct reloc *reloc;
+
+ sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
+ if (!sec)
+ return;
+
+ list_for_each_entry(reloc, &sec->reloc_list, list) {
+ switch (reloc->sym->type) {
+ case STT_FUNC:
+ func = reloc->sym;
+ break;
+
+ case STT_SECTION:
+ func = find_func_by_offset(reloc->sym->sec, reloc->addend);
+ if (!func)
+ continue;
+ break;
+
+ default:
+ WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
+ continue;
+ }
+
+ func_for_each_insn(file, func, insn)
+ insn->ignore = true;
+ }
+}
+
+/*
+ * This is a whitelist of functions that is allowed to be called with AC set.
+ * The list is meant to be minimal and only contains compiler instrumentation
+ * ABI and a few functions used to implement *_{to,from}_user() functions.
+ *
+ * These functions must not directly change AC, but may PUSHF/POPF.
+ */
+static const char *uaccess_safe_builtin[] = {
+ /* KASAN */
+ "kasan_report",
+ "kasan_check_range",
+ /* KASAN out-of-line */
+ "__asan_loadN_noabort",
+ "__asan_load1_noabort",
+ "__asan_load2_noabort",
+ "__asan_load4_noabort",
+ "__asan_load8_noabort",
+ "__asan_load16_noabort",
+ "__asan_storeN_noabort",
+ "__asan_store1_noabort",
+ "__asan_store2_noabort",
+ "__asan_store4_noabort",
+ "__asan_store8_noabort",
+ "__asan_store16_noabort",
+ "__kasan_check_read",
+ "__kasan_check_write",
+ /* KASAN in-line */
+ "__asan_report_load_n_noabort",
+ "__asan_report_load1_noabort",
+ "__asan_report_load2_noabort",
+ "__asan_report_load4_noabort",
+ "__asan_report_load8_noabort",
+ "__asan_report_load16_noabort",
+ "__asan_report_store_n_noabort",
+ "__asan_report_store1_noabort",
+ "__asan_report_store2_noabort",
+ "__asan_report_store4_noabort",
+ "__asan_report_store8_noabort",
+ "__asan_report_store16_noabort",
+ /* KCSAN */
+ "__kcsan_check_access",
+ "__kcsan_mb",
+ "__kcsan_wmb",
+ "__kcsan_rmb",
+ "__kcsan_release",
+ "kcsan_found_watchpoint",
+ "kcsan_setup_watchpoint",
+ "kcsan_check_scoped_accesses",
+ "kcsan_disable_current",
+ "kcsan_enable_current_nowarn",
+ /* KCSAN/TSAN */
+ "__tsan_func_entry",
+ "__tsan_func_exit",
+ "__tsan_read_range",
+ "__tsan_write_range",
+ "__tsan_read1",
+ "__tsan_read2",
+ "__tsan_read4",
+ "__tsan_read8",
+ "__tsan_read16",
+ "__tsan_write1",
+ "__tsan_write2",
+ "__tsan_write4",
+ "__tsan_write8",
+ "__tsan_write16",
+ "__tsan_read_write1",
+ "__tsan_read_write2",
+ "__tsan_read_write4",
+ "__tsan_read_write8",
+ "__tsan_read_write16",
+ "__tsan_volatile_read1",
+ "__tsan_volatile_read2",
+ "__tsan_volatile_read4",
+ "__tsan_volatile_read8",
+ "__tsan_volatile_read16",
+ "__tsan_volatile_write1",
+ "__tsan_volatile_write2",
+ "__tsan_volatile_write4",
+ "__tsan_volatile_write8",
+ "__tsan_volatile_write16",
+ "__tsan_atomic8_load",
+ "__tsan_atomic16_load",
+ "__tsan_atomic32_load",
+ "__tsan_atomic64_load",
+ "__tsan_atomic8_store",
+ "__tsan_atomic16_store",
+ "__tsan_atomic32_store",
+ "__tsan_atomic64_store",
+ "__tsan_atomic8_exchange",
+ "__tsan_atomic16_exchange",
+ "__tsan_atomic32_exchange",
+ "__tsan_atomic64_exchange",
+ "__tsan_atomic8_fetch_add",
+ "__tsan_atomic16_fetch_add",
+ "__tsan_atomic32_fetch_add",
+ "__tsan_atomic64_fetch_add",
+ "__tsan_atomic8_fetch_sub",
+ "__tsan_atomic16_fetch_sub",
+ "__tsan_atomic32_fetch_sub",
+ "__tsan_atomic64_fetch_sub",
+ "__tsan_atomic8_fetch_and",
+ "__tsan_atomic16_fetch_and",
+ "__tsan_atomic32_fetch_and",
+ "__tsan_atomic64_fetch_and",
+ "__tsan_atomic8_fetch_or",
+ "__tsan_atomic16_fetch_or",
+ "__tsan_atomic32_fetch_or",
+ "__tsan_atomic64_fetch_or",
+ "__tsan_atomic8_fetch_xor",
+ "__tsan_atomic16_fetch_xor",
+ "__tsan_atomic32_fetch_xor",
+ "__tsan_atomic64_fetch_xor",
+ "__tsan_atomic8_fetch_nand",
+ "__tsan_atomic16_fetch_nand",
+ "__tsan_atomic32_fetch_nand",
+ "__tsan_atomic64_fetch_nand",
+ "__tsan_atomic8_compare_exchange_strong",
+ "__tsan_atomic16_compare_exchange_strong",
+ "__tsan_atomic32_compare_exchange_strong",
+ "__tsan_atomic64_compare_exchange_strong",
+ "__tsan_atomic8_compare_exchange_weak",
+ "__tsan_atomic16_compare_exchange_weak",
+ "__tsan_atomic32_compare_exchange_weak",
+ "__tsan_atomic64_compare_exchange_weak",
+ "__tsan_atomic8_compare_exchange_val",
+ "__tsan_atomic16_compare_exchange_val",
+ "__tsan_atomic32_compare_exchange_val",
+ "__tsan_atomic64_compare_exchange_val",
+ "__tsan_atomic_thread_fence",
+ "__tsan_atomic_signal_fence",
+ "__tsan_unaligned_read16",
+ "__tsan_unaligned_write16",
+ /* KCOV */
+ "write_comp_data",
+ "check_kcov_mode",
+ "__sanitizer_cov_trace_pc",
+ "__sanitizer_cov_trace_const_cmp1",
+ "__sanitizer_cov_trace_const_cmp2",
+ "__sanitizer_cov_trace_const_cmp4",
+ "__sanitizer_cov_trace_const_cmp8",
+ "__sanitizer_cov_trace_cmp1",
+ "__sanitizer_cov_trace_cmp2",
+ "__sanitizer_cov_trace_cmp4",
+ "__sanitizer_cov_trace_cmp8",
+ "__sanitizer_cov_trace_switch",
+ /* KMSAN */
+ "kmsan_copy_to_user",
+ "kmsan_report",
+ "kmsan_unpoison_entry_regs",
+ "kmsan_unpoison_memory",
+ "__msan_chain_origin",
+ "__msan_get_context_state",
+ "__msan_instrument_asm_store",
+ "__msan_metadata_ptr_for_load_1",
+ "__msan_metadata_ptr_for_load_2",
+ "__msan_metadata_ptr_for_load_4",
+ "__msan_metadata_ptr_for_load_8",
+ "__msan_metadata_ptr_for_load_n",
+ "__msan_metadata_ptr_for_store_1",
+ "__msan_metadata_ptr_for_store_2",
+ "__msan_metadata_ptr_for_store_4",
+ "__msan_metadata_ptr_for_store_8",
+ "__msan_metadata_ptr_for_store_n",
+ "__msan_poison_alloca",
+ "__msan_warning",
+ /* UBSAN */
+ "ubsan_type_mismatch_common",
+ "__ubsan_handle_type_mismatch",
+ "__ubsan_handle_type_mismatch_v1",
+ "__ubsan_handle_shift_out_of_bounds",
+ /* misc */
+ "csum_partial_copy_generic",
+ "copy_mc_fragile",
+ "copy_mc_fragile_handle_tail",
+ "copy_mc_enhanced_fast_string",
+ "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
+ "clear_user_erms",
+ "clear_user_rep_good",
+ "clear_user_original",
+ NULL
+};
+
+static void add_uaccess_safe(struct objtool_file *file)
+{
+ struct symbol *func;
+ const char **name;
+
+ if (!opts.uaccess)
+ return;
+
+ for (name = uaccess_safe_builtin; *name; name++) {
+ func = find_symbol_by_name(file->elf, *name);
+ if (!func)
+ continue;
+
+ func->uaccess_safe = true;
+ }
+}
+
+/*
+ * FIXME: For now, just ignore any alternatives which add retpolines. This is
+ * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
+ * But it at least allows objtool to understand the control flow *around* the
+ * retpoline.
+ */
+static int add_ignore_alternatives(struct objtool_file *file)
+{
+ struct section *sec;
+ struct reloc *reloc;
+ struct instruction *insn;
+
+ sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
+ if (!sec)
+ return 0;
+
+ list_for_each_entry(reloc, &sec->reloc_list, list) {
+ if (reloc->sym->type != STT_SECTION) {
+ WARN("unexpected relocation symbol type in %s", sec->name);
+ return -1;
+ }
+
+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
+ if (!insn) {
+ WARN("bad .discard.ignore_alts entry");
+ return -1;
+ }
+
+ insn->ignore_alts = true;
+ }
+
+ return 0;
+}
+
+/*
+ * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
+ * will be added to the .retpoline_sites section.
+ */
+__weak bool arch_is_retpoline(struct symbol *sym)
+{
+ return false;
+}
+
+/*
+ * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
+ * will be added to the .return_sites section.
+ */
+__weak bool arch_is_rethunk(struct symbol *sym)
+{
+ return false;
+}
+
+/*
+ * Symbols that are embedded inside other instructions, because sometimes crazy
+ * code exists. These are mostly ignored for validation purposes.
+ */
+__weak bool arch_is_embedded_insn(struct symbol *sym)
+{
+ return false;
+}
+
+#define NEGATIVE_RELOC ((void *)-1L)
+
+static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
+{
+ if (insn->reloc == NEGATIVE_RELOC)
+ return NULL;
+
+ if (!insn->reloc) {
+ if (!file)
+ return NULL;
+
+ insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
+ insn->offset, insn->len);
+ if (!insn->reloc) {
+ insn->reloc = NEGATIVE_RELOC;
+ return NULL;
+ }
+ }
+
+ return insn->reloc;
+}
+
+static void remove_insn_ops(struct instruction *insn)
+{
+ struct stack_op *op, *tmp;
+
+ list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
+ list_del(&op->list);
+ free(op);
+ }
+}
+
+static void annotate_call_site(struct objtool_file *file,
+ struct instruction *insn, bool sibling)
+{
+ struct reloc *reloc = insn_reloc(file, insn);
+ struct symbol *sym = insn->call_dest;
+
+ if (!sym)
+ sym = reloc->sym;
+
+ /*
+ * Alternative replacement code is just template code which is
+ * sometimes copied to the original instruction. For now, don't
+ * annotate it. (In the future we might consider annotating the
+ * original instruction if/when it ever makes sense to do so.)
+ */
+ if (!strcmp(insn->sec->name, ".altinstr_replacement"))
+ return;
+
+ if (sym->static_call_tramp) {
+ list_add_tail(&insn->call_node, &file->static_call_list);
+ return;
+ }
+
+ if (sym->retpoline_thunk) {
+ list_add_tail(&insn->call_node, &file->retpoline_call_list);
+ return;
+ }
+
+ /*
+ * Many compilers cannot disable KCOV or sanitizer calls with a function
+ * attribute so they need a little help, NOP out any such calls from
+ * noinstr text.
+ */
+ if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
+ if (reloc) {
+ reloc->type = R_NONE;
+ elf_write_reloc(file->elf, reloc);
+ }
+
+ elf_write_insn(file->elf, insn->sec,
+ insn->offset, insn->len,
+ sibling ? arch_ret_insn(insn->len)
+ : arch_nop_insn(insn->len));
+
+ insn->type = sibling ? INSN_RETURN : INSN_NOP;
+
+ if (sibling) {
+ /*
+ * We've replaced the tail-call JMP insn by two new
+ * insn: RET; INT3, except we only have a single struct
+ * insn here. Mark it retpoline_safe to avoid the SLS
+ * warning, instead of adding another insn.
+ */
+ insn->retpoline_safe = true;
+ }
+
+ return;
+ }
+
+ if (opts.mcount && sym->fentry) {
+ if (sibling)
+ WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset);
+
+ if (reloc) {
+ reloc->type = R_NONE;
+ elf_write_reloc(file->elf, reloc);
+ }
+
+ elf_write_insn(file->elf, insn->sec,
+ insn->offset, insn->len,
+ arch_nop_insn(insn->len));
+
+ insn->type = INSN_NOP;
+
+ list_add_tail(&insn->call_node, &file->mcount_loc_list);
+ return;
+ }
+
+ if (!sibling && dead_end_function(file, sym))
+ insn->dead_end = true;
+}
+
+static void add_call_dest(struct objtool_file *file, struct instruction *insn,
+ struct symbol *dest, bool sibling)
+{
+ insn->call_dest = dest;
+ if (!dest)
+ return;
+
+ /*
+ * Whatever stack impact regular CALLs have, should be undone
+ * by the RETURN of the called function.
+ *
+ * Annotated intra-function calls retain the stack_ops but
+ * are converted to JUMP, see read_intra_function_calls().
+ */
+ remove_insn_ops(insn);
+
+ annotate_call_site(file, insn, sibling);
+}
+
+static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
+{
+ /*
+ * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
+ * so convert them accordingly.
+ */
+ switch (insn->type) {
+ case INSN_CALL:
+ insn->type = INSN_CALL_DYNAMIC;
+ break;
+ case INSN_JUMP_UNCONDITIONAL:
+ insn->type = INSN_JUMP_DYNAMIC;
+ break;
+ case INSN_JUMP_CONDITIONAL:
+ insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
+ break;
+ default:
+ return;
+ }
+
+ insn->retpoline_safe = true;
+
+ /*
+ * Whatever stack impact regular CALLs have, should be undone
+ * by the RETURN of the called function.
+ *
+ * Annotated intra-function calls retain the stack_ops but
+ * are converted to JUMP, see read_intra_function_calls().
+ */
+ remove_insn_ops(insn);
+
+ annotate_call_site(file, insn, false);
+}
+
+static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
+{
+ /*
+ * Return thunk tail calls are really just returns in disguise,
+ * so convert them accordingly.
+ */
+ insn->type = INSN_RETURN;
+ insn->retpoline_safe = true;
+
+ if (add)
+ list_add_tail(&insn->call_node, &file->return_thunk_list);
+}
+
+static bool same_function(struct instruction *insn1, struct instruction *insn2)
+{
+ return insn1->func->pfunc == insn2->func->pfunc;
+}
+
+static bool is_first_func_insn(struct objtool_file *file, struct instruction *insn)
+{
+ if (insn->offset == insn->func->offset)
+ return true;
+
+ if (opts.ibt) {
+ struct instruction *prev = prev_insn_same_sym(file, insn);
+
+ if (prev && prev->type == INSN_ENDBR &&
+ insn->offset == insn->func->offset + prev->len)
+ return true;
+ }
+
+ return false;
+}
+
+/*
+ * Find the destination instructions for all jumps.
+ */
+static int add_jump_destinations(struct objtool_file *file)
+{
+ struct instruction *insn, *jump_dest;
+ struct reloc *reloc;
+ struct section *dest_sec;
+ unsigned long dest_off;
+
+ for_each_insn(file, insn) {
+ if (insn->jump_dest) {
+ /*
+ * handle_group_alt() may have previously set
+ * 'jump_dest' for some alternatives.
+ */
+ continue;
+ }
+ if (!is_static_jump(insn))
+ continue;
+
+ reloc = insn_reloc(file, insn);
+ if (!reloc) {
+ dest_sec = insn->sec;
+ dest_off = arch_jump_destination(insn);
+ } else if (reloc->sym->type == STT_SECTION) {
+ dest_sec = reloc->sym->sec;
+ dest_off = arch_dest_reloc_offset(reloc->addend);
+ } else if (reloc->sym->retpoline_thunk) {
+ add_retpoline_call(file, insn);
+ continue;
+ } else if (reloc->sym->return_thunk) {
+ add_return_call(file, insn, true);
+ continue;
+ } else if (insn->func) {
+ /*
+ * External sibling call or internal sibling call with
+ * STT_FUNC reloc.
+ */
+ add_call_dest(file, insn, reloc->sym, true);
+ continue;
+ } else if (reloc->sym->sec->idx) {
+ dest_sec = reloc->sym->sec;
+ dest_off = reloc->sym->sym.st_value +
+ arch_dest_reloc_offset(reloc->addend);
+ } else {
+ /* non-func asm code jumping to another file */
+ continue;
+ }
+
+ jump_dest = find_insn(file, dest_sec, dest_off);
+ if (!jump_dest) {
+ struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
+
+ /*
+ * This is a special case for retbleed_untrain_ret().
+ * It jumps to __x86_return_thunk(), but objtool
+ * can't find the thunk's starting RET
+ * instruction, because the RET is also in the
+ * middle of another instruction. Objtool only
+ * knows about the outer instruction.
+ */
+ if (sym && sym->embedded_insn) {
+ add_return_call(file, insn, false);
+ continue;
+ }
+
+ WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
+ insn->sec, insn->offset, dest_sec->name,
+ dest_off);
+ return -1;
+ }
+
+ /*
+ * Cross-function jump.
+ */
+ if (insn->func && jump_dest->func &&
+ insn->func != jump_dest->func) {
+
+ /*
+ * For GCC 8+, create parent/child links for any cold
+ * subfunctions. This is _mostly_ redundant with a
+ * similar initialization in read_symbols().
+ *
+ * If a function has aliases, we want the *first* such
+ * function in the symbol table to be the subfunction's
+ * parent. In that case we overwrite the
+ * initialization done in read_symbols().
+ *
+ * However this code can't completely replace the
+ * read_symbols() code because this doesn't detect the
+ * case where the parent function's only reference to a
+ * subfunction is through a jump table.
+ */
+ if (!strstr(insn->func->name, ".cold") &&
+ strstr(jump_dest->func->name, ".cold")) {
+ insn->func->cfunc = jump_dest->func;
+ jump_dest->func->pfunc = insn->func;
+
+ } else if (!same_function(insn, jump_dest) &&
+ is_first_func_insn(file, jump_dest)) {
+ /*
+ * Internal sibling call without reloc or with
+ * STT_SECTION reloc.
+ */
+ add_call_dest(file, insn, jump_dest->func, true);
+ continue;
+ }
+ }
+
+ insn->jump_dest = jump_dest;
+ }
+
+ return 0;
+}
+
+static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
+{
+ struct symbol *call_dest;
+
+ call_dest = find_func_by_offset(sec, offset);
+ if (!call_dest)
+ call_dest = find_symbol_by_offset(sec, offset);
+
+ return call_dest;
+}
+
+/*
+ * Find the destination instructions for all calls.
+ */
+static int add_call_destinations(struct objtool_file *file)
+{
+ struct instruction *insn;
+ unsigned long dest_off;
+ struct symbol *dest;
+ struct reloc *reloc;
+
+ for_each_insn(file, insn) {
+ if (insn->type != INSN_CALL)
+ continue;
+
+ reloc = insn_reloc(file, insn);
+ if (!reloc) {
+ dest_off = arch_jump_destination(insn);
+ dest = find_call_destination(insn->sec, dest_off);
+
+ add_call_dest(file, insn, dest, false);
+
+ if (insn->ignore)
+ continue;
+
+ if (!insn->call_dest) {
+ WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset);
+ return -1;
+ }
+
+ if (insn->func && insn->call_dest->type != STT_FUNC) {
+ WARN_FUNC("unsupported call to non-function",
+ insn->sec, insn->offset);
+ return -1;
+ }
+
+ } else if (reloc->sym->type == STT_SECTION) {
+ dest_off = arch_dest_reloc_offset(reloc->addend);
+ dest = find_call_destination(reloc->sym->sec, dest_off);
+ if (!dest) {
+ WARN_FUNC("can't find call dest symbol at %s+0x%lx",
+ insn->sec, insn->offset,
+ reloc->sym->sec->name,
+ dest_off);
+ return -1;
+ }
+
+ add_call_dest(file, insn, dest, false);
+
+ } else if (reloc->sym->retpoline_thunk) {
+ add_retpoline_call(file, insn);
+
+ } else
+ add_call_dest(file, insn, reloc->sym, false);
+ }
+
+ return 0;
+}
+
+/*
+ * The .alternatives section requires some extra special care over and above
+ * other special sections because alternatives are patched in place.
+ */
+static int handle_group_alt(struct objtool_file *file,
+ struct special_alt *special_alt,
+ struct instruction *orig_insn,
+ struct instruction **new_insn)
+{
+ struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL;
+ struct alt_group *orig_alt_group, *new_alt_group;
+ unsigned long dest_off;
+
+
+ orig_alt_group = malloc(sizeof(*orig_alt_group));
+ if (!orig_alt_group) {
+ WARN("malloc failed");
+ return -1;
+ }
+ orig_alt_group->cfi = calloc(special_alt->orig_len,
+ sizeof(struct cfi_state *));
+ if (!orig_alt_group->cfi) {
+ WARN("calloc failed");
+ return -1;
+ }
+
+ last_orig_insn = NULL;
+ insn = orig_insn;
+ sec_for_each_insn_from(file, insn) {
+ if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
+ break;
+
+ insn->alt_group = orig_alt_group;
+ last_orig_insn = insn;
+ }
+ orig_alt_group->orig_group = NULL;
+ orig_alt_group->first_insn = orig_insn;
+ orig_alt_group->last_insn = last_orig_insn;
+
+
+ new_alt_group = malloc(sizeof(*new_alt_group));
+ if (!new_alt_group) {
+ WARN("malloc failed");
+ return -1;
+ }
+
+ if (special_alt->new_len < special_alt->orig_len) {
+ /*
+ * Insert a fake nop at the end to make the replacement
+ * alt_group the same size as the original. This is needed to
+ * allow propagate_alt_cfi() to do its magic. When the last
+ * instruction affects the stack, the instruction after it (the
+ * nop) will propagate the new state to the shared CFI array.
+ */
+ nop = malloc(sizeof(*nop));
+ if (!nop) {
+ WARN("malloc failed");
+ return -1;
+ }
+ memset(nop, 0, sizeof(*nop));
+ INIT_LIST_HEAD(&nop->alts);
+ INIT_LIST_HEAD(&nop->stack_ops);
+
+ nop->sec = special_alt->new_sec;
+ nop->offset = special_alt->new_off + special_alt->new_len;
+ nop->len = special_alt->orig_len - special_alt->new_len;
+ nop->type = INSN_NOP;
+ nop->func = orig_insn->func;
+ nop->alt_group = new_alt_group;
+ nop->ignore = orig_insn->ignore_alts;
+ }
+
+ if (!special_alt->new_len) {
+ *new_insn = nop;
+ goto end;
+ }
+
+ insn = *new_insn;
+ sec_for_each_insn_from(file, insn) {
+ struct reloc *alt_reloc;
+
+ if (insn->offset >= special_alt->new_off + special_alt->new_len)
+ break;
+
+ last_new_insn = insn;
+
+ insn->ignore = orig_insn->ignore_alts;
+ insn->func = orig_insn->func;
+ insn->alt_group = new_alt_group;
+
+ /*
+ * Since alternative replacement code is copy/pasted by the
+ * kernel after applying relocations, generally such code can't
+ * have relative-address relocation references to outside the
+ * .altinstr_replacement section, unless the arch's
+ * alternatives code can adjust the relative offsets
+ * accordingly.
+ */
+ alt_reloc = insn_reloc(file, insn);
+ if (alt_reloc &&
+ !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
+
+ WARN_FUNC("unsupported relocation in alternatives section",
+ insn->sec, insn->offset);
+ return -1;
+ }
+
+ if (!is_static_jump(insn))
+ continue;
+
+ if (!insn->immediate)
+ continue;
+
+ dest_off = arch_jump_destination(insn);
+ if (dest_off == special_alt->new_off + special_alt->new_len) {
+ insn->jump_dest = next_insn_same_sec(file, last_orig_insn);
+ if (!insn->jump_dest) {
+ WARN_FUNC("can't find alternative jump destination",
+ insn->sec, insn->offset);
+ return -1;
+ }
+ }
+ }
+
+ if (!last_new_insn) {
+ WARN_FUNC("can't find last new alternative instruction",
+ special_alt->new_sec, special_alt->new_off);
+ return -1;
+ }
+
+ if (nop)
+ list_add(&nop->list, &last_new_insn->list);
+end:
+ new_alt_group->orig_group = orig_alt_group;
+ new_alt_group->first_insn = *new_insn;
+ new_alt_group->last_insn = nop ? : last_new_insn;
+ new_alt_group->cfi = orig_alt_group->cfi;
+ return 0;
+}
+
+/*
+ * A jump table entry can either convert a nop to a jump or a jump to a nop.
+ * If the original instruction is a jump, make the alt entry an effective nop
+ * by just skipping the original instruction.
+ */
+static int handle_jump_alt(struct objtool_file *file,
+ struct special_alt *special_alt,
+ struct instruction *orig_insn,
+ struct instruction **new_insn)
+{
+ if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
+ orig_insn->type != INSN_NOP) {
+
+ WARN_FUNC("unsupported instruction at jump label",
+ orig_insn->sec, orig_insn->offset);
+ return -1;
+ }
+
+ if (opts.hack_jump_label && special_alt->key_addend & 2) {
+ struct reloc *reloc = insn_reloc(file, orig_insn);
+
+ if (reloc) {
+ reloc->type = R_NONE;
+ elf_write_reloc(file->elf, reloc);
+ }
+ elf_write_insn(file->elf, orig_insn->sec,
+ orig_insn->offset, orig_insn->len,
+ arch_nop_insn(orig_insn->len));
+ orig_insn->type = INSN_NOP;
+ }
+
+ if (orig_insn->type == INSN_NOP) {
+ if (orig_insn->len == 2)
+ file->jl_nop_short++;
+ else
+ file->jl_nop_long++;
+
+ return 0;
+ }
+
+ if (orig_insn->len == 2)
+ file->jl_short++;
+ else
+ file->jl_long++;
+
+ *new_insn = list_next_entry(orig_insn, list);
+ return 0;
+}
+
+/*
+ * Read all the special sections which have alternate instructions which can be
+ * patched in or redirected to at runtime. Each instruction having alternate
+ * instruction(s) has them added to its insn->alts list, which will be
+ * traversed in validate_branch().
+ */
+static int add_special_section_alts(struct objtool_file *file)
+{
+ struct list_head special_alts;
+ struct instruction *orig_insn, *new_insn;
+ struct special_alt *special_alt, *tmp;
+ struct alternative *alt;
+ int ret;
+
+ ret = special_get_alts(file->elf, &special_alts);
+ if (ret)
+ return ret;
+
+ list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
+
+ orig_insn = find_insn(file, special_alt->orig_sec,
+ special_alt->orig_off);
+ if (!orig_insn) {
+ WARN_FUNC("special: can't find orig instruction",
+ special_alt->orig_sec, special_alt->orig_off);
+ ret = -1;
+ goto out;
+ }
+
+ new_insn = NULL;
+ if (!special_alt->group || special_alt->new_len) {
+ new_insn = find_insn(file, special_alt->new_sec,
+ special_alt->new_off);
+ if (!new_insn) {
+ WARN_FUNC("special: can't find new instruction",
+ special_alt->new_sec,
+ special_alt->new_off);
+ ret = -1;
+ goto out;
+ }
+ }
+
+ if (special_alt->group) {
+ if (!special_alt->orig_len) {
+ WARN_FUNC("empty alternative entry",
+ orig_insn->sec, orig_insn->offset);
+ continue;
+ }
+
+ ret = handle_group_alt(file, special_alt, orig_insn,
+ &new_insn);
+ if (ret)
+ goto out;
+ } else if (special_alt->jump_or_nop) {
+ ret = handle_jump_alt(file, special_alt, orig_insn,
+ &new_insn);
+ if (ret)
+ goto out;
+ }
+
+ alt = malloc(sizeof(*alt));
+ if (!alt) {
+ WARN("malloc failed");
+ ret = -1;
+ goto out;
+ }
+
+ alt->insn = new_insn;
+ alt->skip_orig = special_alt->skip_orig;
+ orig_insn->ignore_alts |= special_alt->skip_alt;
+ list_add_tail(&alt->list, &orig_insn->alts);
+
+ list_del(&special_alt->list);
+ free(special_alt);
+ }
+
+ if (opts.stats) {
+ printf("jl\\\tNOP\tJMP\n");
+ printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
+ printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
+ }
+
+out:
+ return ret;
+}
+
+static int add_jump_table(struct objtool_file *file, struct instruction *insn,
+ struct reloc *table)
+{
+ struct reloc *reloc = table;
+ struct instruction *dest_insn;
+ struct alternative *alt;
+ struct symbol *pfunc = insn->func->pfunc;
+ unsigned int prev_offset = 0;
+
+ /*
+ * Each @reloc is a switch table relocation which points to the target
+ * instruction.
+ */
+ list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
+
+ /* Check for the end of the table: */
+ if (reloc != table && reloc->jump_table_start)
+ break;
+
+ /* Make sure the table entries are consecutive: */
+ if (prev_offset && reloc->offset != prev_offset + 8)
+ break;
+
+ /* Detect function pointers from contiguous objects: */
+ if (reloc->sym->sec == pfunc->sec &&
+ reloc->addend == pfunc->offset)
+ break;
+
+ dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
+ if (!dest_insn)
+ break;
+
+ /* Make sure the destination is in the same function: */
+ if (!dest_insn->func || dest_insn->func->pfunc != pfunc)
+ break;
+
+ alt = malloc(sizeof(*alt));
+ if (!alt) {
+ WARN("malloc failed");
+ return -1;
+ }
+
+ alt->insn = dest_insn;
+ list_add_tail(&alt->list, &insn->alts);
+ prev_offset = reloc->offset;
+ }
+
+ if (!prev_offset) {
+ WARN_FUNC("can't find switch jump table",
+ insn->sec, insn->offset);
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * find_jump_table() - Given a dynamic jump, find the switch jump table
+ * associated with it.
+ */
+static struct reloc *find_jump_table(struct objtool_file *file,
+ struct symbol *func,
+ struct instruction *insn)
+{
+ struct reloc *table_reloc;
+ struct instruction *dest_insn, *orig_insn = insn;
+
+ /*
+ * Backward search using the @first_jump_src links, these help avoid
+ * much of the 'in between' code. Which avoids us getting confused by
+ * it.
+ */
+ for (;
+ insn && insn->func && insn->func->pfunc == func;
+ insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
+
+ if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
+ break;
+
+ /* allow small jumps within the range */
+ if (insn->type == INSN_JUMP_UNCONDITIONAL &&
+ insn->jump_dest &&
+ (insn->jump_dest->offset <= insn->offset ||
+ insn->jump_dest->offset > orig_insn->offset))
+ break;
+
+ table_reloc = arch_find_switch_table(file, insn);
+ if (!table_reloc)
+ continue;
+ dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
+ if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func)
+ continue;
+
+ return table_reloc;
+ }
+
+ return NULL;
+}
+
+/*
+ * First pass: Mark the head of each jump table so that in the next pass,
+ * we know when a given jump table ends and the next one starts.
+ */
+static void mark_func_jump_tables(struct objtool_file *file,
+ struct symbol *func)
+{
+ struct instruction *insn, *last = NULL;
+ struct reloc *reloc;
+
+ func_for_each_insn(file, func, insn) {
+ if (!last)
+ last = insn;
+
+ /*
+ * Store back-pointers for unconditional forward jumps such
+ * that find_jump_table() can back-track using those and
+ * avoid some potentially confusing code.
+ */
+ if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
+ insn->offset > last->offset &&
+ insn->jump_dest->offset > insn->offset &&
+ !insn->jump_dest->first_jump_src) {
+
+ insn->jump_dest->first_jump_src = insn;
+ last = insn->jump_dest;
+ }
+
+ if (insn->type != INSN_JUMP_DYNAMIC)
+ continue;
+
+ reloc = find_jump_table(file, func, insn);
+ if (reloc) {
+ reloc->jump_table_start = true;
+ insn->jump_table = reloc;
+ }
+ }
+}
+
+static int add_func_jump_tables(struct objtool_file *file,
+ struct symbol *func)
+{
+ struct instruction *insn;
+ int ret;
+
+ func_for_each_insn(file, func, insn) {
+ if (!insn->jump_table)
+ continue;
+
+ ret = add_jump_table(file, insn, insn->jump_table);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+/*
+ * For some switch statements, gcc generates a jump table in the .rodata
+ * section which contains a list of addresses within the function to jump to.
+ * This finds these jump tables and adds them to the insn->alts lists.
+ */
+static int add_jump_table_alts(struct objtool_file *file)
+{
+ struct section *sec;
+ struct symbol *func;
+ int ret;
+
+ if (!file->rodata)
+ return 0;
+
+ for_each_sec(file, sec) {
+ list_for_each_entry(func, &sec->symbol_list, list) {
+ if (func->type != STT_FUNC)
+ continue;
+
+ mark_func_jump_tables(file, func);
+ ret = add_func_jump_tables(file, func);
+ if (ret)
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+static void set_func_state(struct cfi_state *state)
+{
+ state->cfa = initial_func_cfi.cfa;
+ memcpy(&state->regs, &initial_func_cfi.regs,
+ CFI_NUM_REGS * sizeof(struct cfi_reg));
+ state->stack_size = initial_func_cfi.cfa.offset;
+}
+
+static int read_unwind_hints(struct objtool_file *file)
+{
+ struct cfi_state cfi = init_cfi;
+ struct section *sec, *relocsec;
+ struct unwind_hint *hint;
+ struct instruction *insn;
+ struct reloc *reloc;
+ int i;
+
+ sec = find_section_by_name(file->elf, ".discard.unwind_hints");
+ if (!sec)
+ return 0;
+
+ relocsec = sec->reloc;
+ if (!relocsec) {
+ WARN("missing .rela.discard.unwind_hints section");
+ return -1;
+ }
+
+ if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
+ WARN("struct unwind_hint size mismatch");
+ return -1;
+ }
+
+ file->hints = true;
+
+ for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
+ hint = (struct unwind_hint *)sec->data->d_buf + i;
+
+ reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
+ if (!reloc) {
+ WARN("can't find reloc for unwind_hints[%d]", i);
+ return -1;
+ }
+
+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
+ if (!insn) {
+ WARN("can't find insn for unwind_hints[%d]", i);
+ return -1;
+ }
+
+ insn->hint = true;
+
+ if (hint->type == UNWIND_HINT_TYPE_SAVE) {
+ insn->hint = false;
+ insn->save = true;
+ continue;
+ }
+
+ if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
+ insn->restore = true;
+ continue;
+ }
+
+ if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
+ struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
+
+ if (sym && sym->bind == STB_GLOBAL) {
+ if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
+ WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR",
+ insn->sec, insn->offset);
+ }
+
+ insn->entry = 1;
+ }
+ }
+
+ if (hint->type == UNWIND_HINT_TYPE_ENTRY) {
+ hint->type = UNWIND_HINT_TYPE_CALL;
+ insn->entry = 1;
+ }
+
+ if (hint->type == UNWIND_HINT_TYPE_FUNC) {
+ insn->cfi = &func_cfi;
+ continue;
+ }
+
+ if (insn->cfi)
+ cfi = *(insn->cfi);
+
+ if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
+ WARN_FUNC("unsupported unwind_hint sp base reg %d",
+ insn->sec, insn->offset, hint->sp_reg);
+ return -1;
+ }
+
+ cfi.cfa.offset = bswap_if_needed(hint->sp_offset);
+ cfi.type = hint->type;
+ cfi.end = hint->end;
+
+ insn->cfi = cfi_hash_find_or_add(&cfi);
+ }
+
+ return 0;
+}
+
+static int read_noendbr_hints(struct objtool_file *file)
+{
+ struct section *sec;
+ struct instruction *insn;
+ struct reloc *reloc;
+
+ sec = find_section_by_name(file->elf, ".rela.discard.noendbr");
+ if (!sec)
+ return 0;
+
+ list_for_each_entry(reloc, &sec->reloc_list, list) {
+ insn = find_insn(file, reloc->sym->sec, reloc->sym->offset + reloc->addend);
+ if (!insn) {
+ WARN("bad .discard.noendbr entry");
+ return -1;
+ }
+
+ insn->noendbr = 1;
+ }
+
+ return 0;
+}
+
+static int read_retpoline_hints(struct objtool_file *file)
+{
+ struct section *sec;
+ struct instruction *insn;
+ struct reloc *reloc;
+
+ sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
+ if (!sec)
+ return 0;
+
+ list_for_each_entry(reloc, &sec->reloc_list, list) {
+ if (reloc->sym->type != STT_SECTION) {
+ WARN("unexpected relocation symbol type in %s", sec->name);
+ return -1;
+ }
+
+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
+ if (!insn) {
+ WARN("bad .discard.retpoline_safe entry");
+ return -1;
+ }
+
+ if (insn->type != INSN_JUMP_DYNAMIC &&
+ insn->type != INSN_CALL_DYNAMIC &&
+ insn->type != INSN_RETURN &&
+ insn->type != INSN_NOP) {
+ WARN_FUNC("retpoline_safe hint not an indirect jump/call/ret/nop",
+ insn->sec, insn->offset);
+ return -1;
+ }
+
+ insn->retpoline_safe = true;
+ }
+
+ return 0;
+}
+
+static int read_instr_hints(struct objtool_file *file)
+{
+ struct section *sec;
+ struct instruction *insn;
+ struct reloc *reloc;
+
+ sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
+ if (!sec)
+ return 0;
+
+ list_for_each_entry(reloc, &sec->reloc_list, list) {
+ if (reloc->sym->type != STT_SECTION) {
+ WARN("unexpected relocation symbol type in %s", sec->name);
+ return -1;
+ }
+
+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
+ if (!insn) {
+ WARN("bad .discard.instr_end entry");
+ return -1;
+ }
+
+ insn->instr--;
+ }
+
+ sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
+ if (!sec)
+ return 0;
+
+ list_for_each_entry(reloc, &sec->reloc_list, list) {
+ if (reloc->sym->type != STT_SECTION) {
+ WARN("unexpected relocation symbol type in %s", sec->name);
+ return -1;
+ }
+
+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
+ if (!insn) {
+ WARN("bad .discard.instr_begin entry");
+ return -1;
+ }
+
+ insn->instr++;
+ }
+
+ return 0;
+}
+
+static int read_intra_function_calls(struct objtool_file *file)
+{
+ struct instruction *insn;
+ struct section *sec;
+ struct reloc *reloc;
+
+ sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
+ if (!sec)
+ return 0;
+
+ list_for_each_entry(reloc, &sec->reloc_list, list) {
+ unsigned long dest_off;
+
+ if (reloc->sym->type != STT_SECTION) {
+ WARN("unexpected relocation symbol type in %s",
+ sec->name);
+ return -1;
+ }
+
+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
+ if (!insn) {
+ WARN("bad .discard.intra_function_call entry");
+ return -1;
+ }
+
+ if (insn->type != INSN_CALL) {
+ WARN_FUNC("intra_function_call not a direct call",
+ insn->sec, insn->offset);
+ return -1;
+ }
+
+ /*
+ * Treat intra-function CALLs as JMPs, but with a stack_op.
+ * See add_call_destinations(), which strips stack_ops from
+ * normal CALLs.
+ */
+ insn->type = INSN_JUMP_UNCONDITIONAL;
+
+ dest_off = arch_jump_destination(insn);
+ insn->jump_dest = find_insn(file, insn->sec, dest_off);
+ if (!insn->jump_dest) {
+ WARN_FUNC("can't find call dest at %s+0x%lx",
+ insn->sec, insn->offset,
+ insn->sec->name, dest_off);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * Return true if name matches an instrumentation function, where calls to that
+ * function from noinstr code can safely be removed, but compilers won't do so.
+ */
+static bool is_profiling_func(const char *name)
+{
+ /*
+ * Many compilers cannot disable KCOV with a function attribute.
+ */
+ if (!strncmp(name, "__sanitizer_cov_", 16))
+ return true;
+
+ /*
+ * Some compilers currently do not remove __tsan_func_entry/exit nor
+ * __tsan_atomic_signal_fence (used for barrier instrumentation) with
+ * the __no_sanitize_thread attribute, remove them. Once the kernel's
+ * minimum Clang version is 14.0, this can be removed.
+ */
+ if (!strncmp(name, "__tsan_func_", 12) ||
+ !strcmp(name, "__tsan_atomic_signal_fence"))
+ return true;
+
+ return false;
+}
+
+static int classify_symbols(struct objtool_file *file)
+{
+ struct section *sec;
+ struct symbol *func;
+
+ for_each_sec(file, sec) {
+ list_for_each_entry(func, &sec->symbol_list, list) {
+ if (func->bind != STB_GLOBAL)
+ continue;
+
+ if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
+ strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
+ func->static_call_tramp = true;
+
+ if (arch_is_retpoline(func))
+ func->retpoline_thunk = true;
+
+ if (arch_is_rethunk(func))
+ func->return_thunk = true;
+
+ if (arch_is_embedded_insn(func))
+ func->embedded_insn = true;
+
+ if (!strcmp(func->name, "__fentry__"))
+ func->fentry = true;
+
+ if (is_profiling_func(func->name))
+ func->profiling_func = true;
+ }
+ }
+
+ return 0;
+}
+
+static void mark_rodata(struct objtool_file *file)
+{
+ struct section *sec;
+ bool found = false;
+
+ /*
+ * Search for the following rodata sections, each of which can
+ * potentially contain jump tables:
+ *
+ * - .rodata: can contain GCC switch tables
+ * - .rodata.<func>: same, if -fdata-sections is being used
+ * - .rodata..c_jump_table: contains C annotated jump tables
+ *
+ * .rodata.str1.* sections are ignored; they don't contain jump tables.
+ */
+ for_each_sec(file, sec) {
+ if (!strncmp(sec->name, ".rodata", 7) &&
+ !strstr(sec->name, ".str1.")) {
+ sec->rodata = true;
+ found = true;
+ }
+ }
+
+ file->rodata = found;
+}
+
+static int decode_sections(struct objtool_file *file)
+{
+ int ret;
+
+ mark_rodata(file);
+
+ ret = init_pv_ops(file);
+ if (ret)
+ return ret;
+
+ ret = decode_instructions(file);
+ if (ret)
+ return ret;
+
+ add_ignores(file);
+ add_uaccess_safe(file);
+
+ ret = add_ignore_alternatives(file);
+ if (ret)
+ return ret;
+
+ /*
+ * Must be before read_unwind_hints() since that needs insn->noendbr.
+ */
+ ret = read_noendbr_hints(file);
+ if (ret)
+ return ret;
+
+ /*
+ * Must be before add_{jump_call}_destination.
+ */
+ ret = classify_symbols(file);
+ if (ret)
+ return ret;
+
+ /*
+ * Must be before add_jump_destinations(), which depends on 'func'
+ * being set for alternatives, to enable proper sibling call detection.
+ */
+ ret = add_special_section_alts(file);
+ if (ret)
+ return ret;
+
+ ret = add_jump_destinations(file);
+ if (ret)
+ return ret;
+
+ /*
+ * Must be before add_call_destination(); it changes INSN_CALL to
+ * INSN_JUMP.
+ */
+ ret = read_intra_function_calls(file);
+ if (ret)
+ return ret;
+
+ ret = add_call_destinations(file);
+ if (ret)
+ return ret;
+
+ /*
+ * Must be after add_call_destinations() such that it can override
+ * dead_end_function() marks.
+ */
+ ret = add_dead_ends(file);
+ if (ret)
+ return ret;
+
+ ret = add_jump_table_alts(file);
+ if (ret)
+ return ret;
+
+ ret = read_unwind_hints(file);
+ if (ret)
+ return ret;
+
+ ret = read_retpoline_hints(file);
+ if (ret)
+ return ret;
+
+ ret = read_instr_hints(file);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+static bool is_special_call(struct instruction *insn)
+{
+ if (insn->type == INSN_CALL) {
+ struct symbol *dest = insn->call_dest;
+
+ if (!dest)
+ return false;
+
+ if (dest->fentry || dest->embedded_insn)
+ return true;
+ }
+
+ return false;
+}
+
+static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
+{
+ struct cfi_state *cfi = &state->cfi;
+ int i;
+
+ if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
+ return true;
+
+ if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
+ return true;
+
+ if (cfi->stack_size != initial_func_cfi.cfa.offset)
+ return true;
+
+ for (i = 0; i < CFI_NUM_REGS; i++) {
+ if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
+ cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
+ return true;
+ }
+
+ return false;
+}
+
+static bool check_reg_frame_pos(const struct cfi_reg *reg,
+ int expected_offset)
+{
+ return reg->base == CFI_CFA &&
+ reg->offset == expected_offset;
+}
+
+static bool has_valid_stack_frame(struct insn_state *state)
+{
+ struct cfi_state *cfi = &state->cfi;
+
+ if (cfi->cfa.base == CFI_BP &&
+ check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
+ check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
+ return true;
+
+ if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
+ return true;
+
+ return false;
+}
+
+static int update_cfi_state_regs(struct instruction *insn,
+ struct cfi_state *cfi,
+ struct stack_op *op)
+{
+ struct cfi_reg *cfa = &cfi->cfa;
+
+ if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
+ return 0;
+
+ /* push */
+ if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
+ cfa->offset += 8;
+
+ /* pop */
+ if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
+ cfa->offset -= 8;
+
+ /* add immediate to sp */
+ if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
+ op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
+ cfa->offset -= op->src.offset;
+
+ return 0;
+}
+
+static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
+{
+ if (arch_callee_saved_reg(reg) &&
+ cfi->regs[reg].base == CFI_UNDEFINED) {
+ cfi->regs[reg].base = base;
+ cfi->regs[reg].offset = offset;
+ }
+}
+
+static void restore_reg(struct cfi_state *cfi, unsigned char reg)
+{
+ cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
+ cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
+}
+
+/*
+ * A note about DRAP stack alignment:
+ *
+ * GCC has the concept of a DRAP register, which is used to help keep track of
+ * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
+ * register. The typical DRAP pattern is:
+ *
+ * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
+ * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
+ * 41 ff 72 f8 pushq -0x8(%r10)
+ * 55 push %rbp
+ * 48 89 e5 mov %rsp,%rbp
+ * (more pushes)
+ * 41 52 push %r10
+ * ...
+ * 41 5a pop %r10
+ * (more pops)
+ * 5d pop %rbp
+ * 49 8d 62 f8 lea -0x8(%r10),%rsp
+ * c3 retq
+ *
+ * There are some variations in the epilogues, like:
+ *
+ * 5b pop %rbx
+ * 41 5a pop %r10
+ * 41 5c pop %r12
+ * 41 5d pop %r13
+ * 41 5e pop %r14
+ * c9 leaveq
+ * 49 8d 62 f8 lea -0x8(%r10),%rsp
+ * c3 retq
+ *
+ * and:
+ *
+ * 4c 8b 55 e8 mov -0x18(%rbp),%r10
+ * 48 8b 5d e0 mov -0x20(%rbp),%rbx
+ * 4c 8b 65 f0 mov -0x10(%rbp),%r12
+ * 4c 8b 6d f8 mov -0x8(%rbp),%r13
+ * c9 leaveq
+ * 49 8d 62 f8 lea -0x8(%r10),%rsp
+ * c3 retq
+ *
+ * Sometimes r13 is used as the DRAP register, in which case it's saved and
+ * restored beforehand:
+ *
+ * 41 55 push %r13
+ * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
+ * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
+ * ...
+ * 49 8d 65 f0 lea -0x10(%r13),%rsp
+ * 41 5d pop %r13
+ * c3 retq
+ */
+static int update_cfi_state(struct instruction *insn,
+ struct instruction *next_insn,
+ struct cfi_state *cfi, struct stack_op *op)
+{
+ struct cfi_reg *cfa = &cfi->cfa;
+ struct cfi_reg *regs = cfi->regs;
+
+ /* stack operations don't make sense with an undefined CFA */
+ if (cfa->base == CFI_UNDEFINED) {
+ if (insn->func) {
+ WARN_FUNC("undefined stack state", insn->sec, insn->offset);
+ return -1;
+ }
+ return 0;
+ }
+
+ if (cfi->type == UNWIND_HINT_TYPE_REGS ||
+ cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
+ return update_cfi_state_regs(insn, cfi, op);
+
+ switch (op->dest.type) {
+
+ case OP_DEST_REG:
+ switch (op->src.type) {
+
+ case OP_SRC_REG:
+ if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
+ cfa->base == CFI_SP &&
+ check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
+
+ /* mov %rsp, %rbp */
+ cfa->base = op->dest.reg;
+ cfi->bp_scratch = false;
+ }
+
+ else if (op->src.reg == CFI_SP &&
+ op->dest.reg == CFI_BP && cfi->drap) {
+
+ /* drap: mov %rsp, %rbp */
+ regs[CFI_BP].base = CFI_BP;
+ regs[CFI_BP].offset = -cfi->stack_size;
+ cfi->bp_scratch = false;
+ }
+
+ else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
+
+ /*
+ * mov %rsp, %reg
+ *
+ * This is needed for the rare case where GCC
+ * does:
+ *
+ * mov %rsp, %rax
+ * ...
+ * mov %rax, %rsp
+ */
+ cfi->vals[op->dest.reg].base = CFI_CFA;
+ cfi->vals[op->dest.reg].offset = -cfi->stack_size;
+ }
+
+ else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
+ (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
+
+ /*
+ * mov %rbp, %rsp
+ *
+ * Restore the original stack pointer (Clang).
+ */
+ cfi->stack_size = -cfi->regs[CFI_BP].offset;
+ }
+
+ else if (op->dest.reg == cfa->base) {
+
+ /* mov %reg, %rsp */
+ if (cfa->base == CFI_SP &&
+ cfi->vals[op->src.reg].base == CFI_CFA) {
+
+ /*
+ * This is needed for the rare case
+ * where GCC does something dumb like:
+ *
+ * lea 0x8(%rsp), %rcx
+ * ...
+ * mov %rcx, %rsp
+ */
+ cfa->offset = -cfi->vals[op->src.reg].offset;
+ cfi->stack_size = cfa->offset;
+
+ } else if (cfa->base == CFI_SP &&
+ cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
+ cfi->vals[op->src.reg].offset == cfa->offset) {
+
+ /*
+ * Stack swizzle:
+ *
+ * 1: mov %rsp, (%[tos])
+ * 2: mov %[tos], %rsp
+ * ...
+ * 3: pop %rsp
+ *
+ * Where:
+ *
+ * 1 - places a pointer to the previous
+ * stack at the Top-of-Stack of the
+ * new stack.
+ *
+ * 2 - switches to the new stack.
+ *
+ * 3 - pops the Top-of-Stack to restore
+ * the original stack.
+ *
+ * Note: we set base to SP_INDIRECT
+ * here and preserve offset. Therefore
+ * when the unwinder reaches ToS it
+ * will dereference SP and then add the
+ * offset to find the next frame, IOW:
+ * (%rsp) + offset.
+ */
+ cfa->base = CFI_SP_INDIRECT;
+
+ } else {
+ cfa->base = CFI_UNDEFINED;
+ cfa->offset = 0;
+ }
+ }
+
+ else if (op->dest.reg == CFI_SP &&
+ cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
+ cfi->vals[op->src.reg].offset == cfa->offset) {
+
+ /*
+ * The same stack swizzle case 2) as above. But
+ * because we can't change cfa->base, case 3)
+ * will become a regular POP. Pretend we're a
+ * PUSH so things don't go unbalanced.
+ */
+ cfi->stack_size += 8;
+ }
+
+
+ break;
+
+ case OP_SRC_ADD:
+ if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
+
+ /* add imm, %rsp */
+ cfi->stack_size -= op->src.offset;
+ if (cfa->base == CFI_SP)
+ cfa->offset -= op->src.offset;
+ break;
+ }
+
+ if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
+
+ /* lea disp(%rbp), %rsp */
+ cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
+ break;
+ }
+
+ if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
+
+ /* drap: lea disp(%rsp), %drap */
+ cfi->drap_reg = op->dest.reg;
+
+ /*
+ * lea disp(%rsp), %reg
+ *
+ * This is needed for the rare case where GCC
+ * does something dumb like:
+ *
+ * lea 0x8(%rsp), %rcx
+ * ...
+ * mov %rcx, %rsp
+ */
+ cfi->vals[op->dest.reg].base = CFI_CFA;
+ cfi->vals[op->dest.reg].offset = \
+ -cfi->stack_size + op->src.offset;
+
+ break;
+ }
+
+ if (cfi->drap && op->dest.reg == CFI_SP &&
+ op->src.reg == cfi->drap_reg) {
+
+ /* drap: lea disp(%drap), %rsp */
+ cfa->base = CFI_SP;
+ cfa->offset = cfi->stack_size = -op->src.offset;
+ cfi->drap_reg = CFI_UNDEFINED;
+ cfi->drap = false;
+ break;
+ }
+
+ if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
+ WARN_FUNC("unsupported stack register modification",
+ insn->sec, insn->offset);
+ return -1;
+ }
+
+ break;
+
+ case OP_SRC_AND:
+ if (op->dest.reg != CFI_SP ||
+ (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
+ (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
+ WARN_FUNC("unsupported stack pointer realignment",
+ insn->sec, insn->offset);
+ return -1;
+ }
+
+ if (cfi->drap_reg != CFI_UNDEFINED) {
+ /* drap: and imm, %rsp */
+ cfa->base = cfi->drap_reg;
+ cfa->offset = cfi->stack_size = 0;
+ cfi->drap = true;
+ }
+
+ /*
+ * Older versions of GCC (4.8ish) realign the stack
+ * without DRAP, with a frame pointer.
+ */
+
+ break;
+
+ case OP_SRC_POP:
+ case OP_SRC_POPF:
+ if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
+
+ /* pop %rsp; # restore from a stack swizzle */
+ cfa->base = CFI_SP;
+ break;
+ }
+
+ if (!cfi->drap && op->dest.reg == cfa->base) {
+
+ /* pop %rbp */
+ cfa->base = CFI_SP;
+ }
+
+ if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
+ op->dest.reg == cfi->drap_reg &&
+ cfi->drap_offset == -cfi->stack_size) {
+
+ /* drap: pop %drap */
+ cfa->base = cfi->drap_reg;
+ cfa->offset = 0;
+ cfi->drap_offset = -1;
+
+ } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
+
+ /* pop %reg */
+ restore_reg(cfi, op->dest.reg);
+ }
+
+ cfi->stack_size -= 8;
+ if (cfa->base == CFI_SP)
+ cfa->offset -= 8;
+
+ break;
+
+ case OP_SRC_REG_INDIRECT:
+ if (!cfi->drap && op->dest.reg == cfa->base &&
+ op->dest.reg == CFI_BP) {
+
+ /* mov disp(%rsp), %rbp */
+ cfa->base = CFI_SP;
+ cfa->offset = cfi->stack_size;
+ }
+
+ if (cfi->drap && op->src.reg == CFI_BP &&
+ op->src.offset == cfi->drap_offset) {
+
+ /* drap: mov disp(%rbp), %drap */
+ cfa->base = cfi->drap_reg;
+ cfa->offset = 0;
+ cfi->drap_offset = -1;
+ }
+
+ if (cfi->drap && op->src.reg == CFI_BP &&
+ op->src.offset == regs[op->dest.reg].offset) {
+
+ /* drap: mov disp(%rbp), %reg */
+ restore_reg(cfi, op->dest.reg);
+
+ } else if (op->src.reg == cfa->base &&
+ op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
+
+ /* mov disp(%rbp), %reg */
+ /* mov disp(%rsp), %reg */
+ restore_reg(cfi, op->dest.reg);
+
+ } else if (op->src.reg == CFI_SP &&
+ op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
+
+ /* mov disp(%rsp), %reg */
+ restore_reg(cfi, op->dest.reg);
+ }
+
+ break;
+
+ default:
+ WARN_FUNC("unknown stack-related instruction",
+ insn->sec, insn->offset);
+ return -1;
+ }
+
+ break;
+
+ case OP_DEST_PUSH:
+ case OP_DEST_PUSHF:
+ cfi->stack_size += 8;
+ if (cfa->base == CFI_SP)
+ cfa->offset += 8;
+
+ if (op->src.type != OP_SRC_REG)
+ break;
+
+ if (cfi->drap) {
+ if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
+
+ /* drap: push %drap */
+ cfa->base = CFI_BP_INDIRECT;
+ cfa->offset = -cfi->stack_size;
+
+ /* save drap so we know when to restore it */
+ cfi->drap_offset = -cfi->stack_size;
+
+ } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
+
+ /* drap: push %rbp */
+ cfi->stack_size = 0;
+
+ } else {
+
+ /* drap: push %reg */
+ save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
+ }
+
+ } else {
+
+ /* push %reg */
+ save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
+ }
+
+ /* detect when asm code uses rbp as a scratch register */
+ if (opts.stackval && insn->func && op->src.reg == CFI_BP &&
+ cfa->base != CFI_BP)
+ cfi->bp_scratch = true;
+ break;
+
+ case OP_DEST_REG_INDIRECT:
+
+ if (cfi->drap) {
+ if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
+
+ /* drap: mov %drap, disp(%rbp) */
+ cfa->base = CFI_BP_INDIRECT;
+ cfa->offset = op->dest.offset;
+
+ /* save drap offset so we know when to restore it */
+ cfi->drap_offset = op->dest.offset;
+ } else {
+
+ /* drap: mov reg, disp(%rbp) */
+ save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
+ }
+
+ } else if (op->dest.reg == cfa->base) {
+
+ /* mov reg, disp(%rbp) */
+ /* mov reg, disp(%rsp) */
+ save_reg(cfi, op->src.reg, CFI_CFA,
+ op->dest.offset - cfi->cfa.offset);
+
+ } else if (op->dest.reg == CFI_SP) {
+
+ /* mov reg, disp(%rsp) */
+ save_reg(cfi, op->src.reg, CFI_CFA,
+ op->dest.offset - cfi->stack_size);
+
+ } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
+
+ /* mov %rsp, (%reg); # setup a stack swizzle. */
+ cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
+ cfi->vals[op->dest.reg].offset = cfa->offset;
+ }
+
+ break;
+
+ case OP_DEST_MEM:
+ if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
+ WARN_FUNC("unknown stack-related memory operation",
+ insn->sec, insn->offset);
+ return -1;
+ }
+
+ /* pop mem */
+ cfi->stack_size -= 8;
+ if (cfa->base == CFI_SP)
+ cfa->offset -= 8;
+
+ break;
+
+ default:
+ WARN_FUNC("unknown stack-related instruction",
+ insn->sec, insn->offset);
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * The stack layouts of alternatives instructions can sometimes diverge when
+ * they have stack modifications. That's fine as long as the potential stack
+ * layouts don't conflict at any given potential instruction boundary.
+ *
+ * Flatten the CFIs of the different alternative code streams (both original
+ * and replacement) into a single shared CFI array which can be used to detect
+ * conflicts and nicely feed a linear array of ORC entries to the unwinder.
+ */
+static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
+{
+ struct cfi_state **alt_cfi;
+ int group_off;
+
+ if (!insn->alt_group)
+ return 0;
+
+ if (!insn->cfi) {
+ WARN("CFI missing");
+ return -1;
+ }
+
+ alt_cfi = insn->alt_group->cfi;
+ group_off = insn->offset - insn->alt_group->first_insn->offset;
+
+ if (!alt_cfi[group_off]) {
+ alt_cfi[group_off] = insn->cfi;
+ } else {
+ if (cficmp(alt_cfi[group_off], insn->cfi)) {
+ WARN_FUNC("stack layout conflict in alternatives",
+ insn->sec, insn->offset);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+static int handle_insn_ops(struct instruction *insn,
+ struct instruction *next_insn,
+ struct insn_state *state)
+{
+ struct stack_op *op;
+
+ list_for_each_entry(op, &insn->stack_ops, list) {
+
+ if (update_cfi_state(insn, next_insn, &state->cfi, op))
+ return 1;
+
+ if (!insn->alt_group)
+ continue;
+
+ if (op->dest.type == OP_DEST_PUSHF) {
+ if (!state->uaccess_stack) {
+ state->uaccess_stack = 1;
+ } else if (state->uaccess_stack >> 31) {
+ WARN_FUNC("PUSHF stack exhausted",
+ insn->sec, insn->offset);
+ return 1;
+ }
+ state->uaccess_stack <<= 1;
+ state->uaccess_stack |= state->uaccess;
+ }
+
+ if (op->src.type == OP_SRC_POPF) {
+ if (state->uaccess_stack) {
+ state->uaccess = state->uaccess_stack & 1;
+ state->uaccess_stack >>= 1;
+ if (state->uaccess_stack == 1)
+ state->uaccess_stack = 0;
+ }
+ }
+ }
+
+ return 0;
+}
+
+static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
+{
+ struct cfi_state *cfi1 = insn->cfi;
+ int i;
+
+ if (!cfi1) {
+ WARN("CFI missing");
+ return false;
+ }
+
+ if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
+
+ WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
+ insn->sec, insn->offset,
+ cfi1->cfa.base, cfi1->cfa.offset,
+ cfi2->cfa.base, cfi2->cfa.offset);
+
+ } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
+ for (i = 0; i < CFI_NUM_REGS; i++) {
+ if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
+ sizeof(struct cfi_reg)))
+ continue;
+
+ WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
+ insn->sec, insn->offset,
+ i, cfi1->regs[i].base, cfi1->regs[i].offset,
+ i, cfi2->regs[i].base, cfi2->regs[i].offset);
+ break;
+ }
+
+ } else if (cfi1->type != cfi2->type) {
+
+ WARN_FUNC("stack state mismatch: type1=%d type2=%d",
+ insn->sec, insn->offset, cfi1->type, cfi2->type);
+
+ } else if (cfi1->drap != cfi2->drap ||
+ (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
+ (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
+
+ WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
+ insn->sec, insn->offset,
+ cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
+ cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
+
+ } else
+ return true;
+
+ return false;
+}
+
+static inline bool func_uaccess_safe(struct symbol *func)
+{
+ if (func)
+ return func->uaccess_safe;
+
+ return false;
+}
+
+static inline const char *call_dest_name(struct instruction *insn)
+{
+ static char pvname[19];
+ struct reloc *rel;
+ int idx;
+
+ if (insn->call_dest)
+ return insn->call_dest->name;
+
+ rel = insn_reloc(NULL, insn);
+ if (rel && !strcmp(rel->sym->name, "pv_ops")) {
+ idx = (rel->addend / sizeof(void *));
+ snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
+ return pvname;
+ }
+
+ return "{dynamic}";
+}
+
+static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
+{
+ struct symbol *target;
+ struct reloc *rel;
+ int idx;
+
+ rel = insn_reloc(file, insn);
+ if (!rel || strcmp(rel->sym->name, "pv_ops"))
+ return false;
+
+ idx = (arch_dest_reloc_offset(rel->addend) / sizeof(void *));
+
+ if (file->pv_ops[idx].clean)
+ return true;
+
+ file->pv_ops[idx].clean = true;
+
+ list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
+ if (!target->sec->noinstr) {
+ WARN("pv_ops[%d]: %s", idx, target->name);
+ file->pv_ops[idx].clean = false;
+ }
+ }
+
+ return file->pv_ops[idx].clean;
+}
+
+static inline bool noinstr_call_dest(struct objtool_file *file,
+ struct instruction *insn,
+ struct symbol *func)
+{
+ /*
+ * We can't deal with indirect function calls at present;
+ * assume they're instrumented.
+ */
+ if (!func) {
+ if (file->pv_ops)
+ return pv_call_dest(file, insn);
+
+ return false;
+ }
+
+ /*
+ * If the symbol is from a noinstr section; we good.
+ */
+ if (func->sec->noinstr)
+ return true;
+
+ /*
+ * The __ubsan_handle_*() calls are like WARN(), they only happen when
+ * something 'BAD' happened. At the risk of taking the machine down,
+ * let them proceed to get the message out.
+ */
+ if (!strncmp(func->name, "__ubsan_handle_", 15))
+ return true;
+
+ return false;
+}
+
+static int validate_call(struct objtool_file *file,
+ struct instruction *insn,
+ struct insn_state *state)
+{
+ if (state->noinstr && state->instr <= 0 &&
+ !noinstr_call_dest(file, insn, insn->call_dest)) {
+ WARN_FUNC("call to %s() leaves .noinstr.text section",
+ insn->sec, insn->offset, call_dest_name(insn));
+ return 1;
+ }
+
+ if (state->uaccess && !func_uaccess_safe(insn->call_dest)) {
+ WARN_FUNC("call to %s() with UACCESS enabled",
+ insn->sec, insn->offset, call_dest_name(insn));
+ return 1;
+ }
+
+ if (state->df) {
+ WARN_FUNC("call to %s() with DF set",
+ insn->sec, insn->offset, call_dest_name(insn));
+ return 1;
+ }
+
+ return 0;
+}
+
+static int validate_sibling_call(struct objtool_file *file,
+ struct instruction *insn,
+ struct insn_state *state)
+{
+ if (has_modified_stack_frame(insn, state)) {
+ WARN_FUNC("sibling call from callable instruction with modified stack frame",
+ insn->sec, insn->offset);
+ return 1;
+ }
+
+ return validate_call(file, insn, state);
+}
+
+static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
+{
+ if (state->noinstr && state->instr > 0) {
+ WARN_FUNC("return with instrumentation enabled",
+ insn->sec, insn->offset);
+ return 1;
+ }
+
+ if (state->uaccess && !func_uaccess_safe(func)) {
+ WARN_FUNC("return with UACCESS enabled",
+ insn->sec, insn->offset);
+ return 1;
+ }
+
+ if (!state->uaccess && func_uaccess_safe(func)) {
+ WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function",
+ insn->sec, insn->offset);
+ return 1;
+ }
+
+ if (state->df) {
+ WARN_FUNC("return with DF set",
+ insn->sec, insn->offset);
+ return 1;
+ }
+
+ if (func && has_modified_stack_frame(insn, state)) {
+ WARN_FUNC("return with modified stack frame",
+ insn->sec, insn->offset);
+ return 1;
+ }
+
+ if (state->cfi.bp_scratch) {
+ WARN_FUNC("BP used as a scratch register",
+ insn->sec, insn->offset);
+ return 1;
+ }
+
+ return 0;
+}
+
+static struct instruction *next_insn_to_validate(struct objtool_file *file,
+ struct instruction *insn)
+{
+ struct alt_group *alt_group = insn->alt_group;
+
+ /*
+ * Simulate the fact that alternatives are patched in-place. When the
+ * end of a replacement alt_group is reached, redirect objtool flow to
+ * the end of the original alt_group.
+ */
+ if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
+ return next_insn_same_sec(file, alt_group->orig_group->last_insn);
+
+ return next_insn_same_sec(file, insn);
+}
+
+/*
+ * Follow the branch starting at the given instruction, and recursively follow
+ * any other branches (jumps). Meanwhile, track the frame pointer state at
+ * each instruction and validate all the rules described in
+ * tools/objtool/Documentation/objtool.txt.
+ */
+static int validate_branch(struct objtool_file *file, struct symbol *func,
+ struct instruction *insn, struct insn_state state)
+{
+ struct alternative *alt;
+ struct instruction *next_insn, *prev_insn = NULL;
+ struct section *sec;
+ u8 visited;
+ int ret;
+
+ sec = insn->sec;
+
+ while (1) {
+ next_insn = next_insn_to_validate(file, insn);
+
+ if (func && insn->func && func != insn->func->pfunc) {
+ /* Ignore KCFI type preambles, which always fall through */
+ if (!strncmp(func->name, "__cfi_", 6))
+ return 0;
+
+ WARN("%s() falls through to next function %s()",
+ func->name, insn->func->name);
+ return 1;
+ }
+
+ if (func && insn->ignore) {
+ WARN_FUNC("BUG: why am I validating an ignored function?",
+ sec, insn->offset);
+ return 1;
+ }
+
+ visited = VISITED_BRANCH << state.uaccess;
+ if (insn->visited & VISITED_BRANCH_MASK) {
+ if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
+ return 1;
+
+ if (insn->visited & visited)
+ return 0;
+ } else {
+ nr_insns_visited++;
+ }
+
+ if (state.noinstr)
+ state.instr += insn->instr;
+
+ if (insn->hint) {
+ if (insn->restore) {
+ struct instruction *save_insn, *i;
+
+ i = insn;
+ save_insn = NULL;
+
+ sym_for_each_insn_continue_reverse(file, func, i) {
+ if (i->save) {
+ save_insn = i;
+ break;
+ }
+ }
+
+ if (!save_insn) {
+ WARN_FUNC("no corresponding CFI save for CFI restore",
+ sec, insn->offset);
+ return 1;
+ }
+
+ if (!save_insn->visited) {
+ WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo",
+ sec, insn->offset);
+ return 1;
+ }
+
+ insn->cfi = save_insn->cfi;
+ nr_cfi_reused++;
+ }
+
+ state.cfi = *insn->cfi;
+ } else {
+ /* XXX track if we actually changed state.cfi */
+
+ if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
+ insn->cfi = prev_insn->cfi;
+ nr_cfi_reused++;
+ } else {
+ insn->cfi = cfi_hash_find_or_add(&state.cfi);
+ }
+ }
+
+ insn->visited |= visited;
+
+ if (propagate_alt_cfi(file, insn))
+ return 1;
+
+ if (!insn->ignore_alts && !list_empty(&insn->alts)) {
+ bool skip_orig = false;
+
+ list_for_each_entry(alt, &insn->alts, list) {
+ if (alt->skip_orig)
+ skip_orig = true;
+
+ ret = validate_branch(file, func, alt->insn, state);
+ if (ret) {
+ if (opts.backtrace)
+ BT_FUNC("(alt)", insn);
+ return ret;
+ }
+ }
+
+ if (skip_orig)
+ return 0;
+ }
+
+ if (handle_insn_ops(insn, next_insn, &state))
+ return 1;
+
+ switch (insn->type) {
+
+ case INSN_RETURN:
+ return validate_return(func, insn, &state);
+
+ case INSN_CALL:
+ case INSN_CALL_DYNAMIC:
+ ret = validate_call(file, insn, &state);
+ if (ret)
+ return ret;
+
+ if (opts.stackval && func && !is_special_call(insn) &&
+ !has_valid_stack_frame(&state)) {
+ WARN_FUNC("call without frame pointer save/setup",
+ sec, insn->offset);
+ return 1;
+ }
+
+ if (insn->dead_end)
+ return 0;
+
+ break;
+
+ case INSN_JUMP_CONDITIONAL:
+ case INSN_JUMP_UNCONDITIONAL:
+ if (is_sibling_call(insn)) {
+ ret = validate_sibling_call(file, insn, &state);
+ if (ret)
+ return ret;
+
+ } else if (insn->jump_dest) {
+ ret = validate_branch(file, func,
+ insn->jump_dest, state);
+ if (ret) {
+ if (opts.backtrace)
+ BT_FUNC("(branch)", insn);
+ return ret;
+ }
+ }
+
+ if (insn->type == INSN_JUMP_UNCONDITIONAL)
+ return 0;
+
+ break;
+
+ case INSN_JUMP_DYNAMIC:
+ case INSN_JUMP_DYNAMIC_CONDITIONAL:
+ if (is_sibling_call(insn)) {
+ ret = validate_sibling_call(file, insn, &state);
+ if (ret)
+ return ret;
+ }
+
+ if (insn->type == INSN_JUMP_DYNAMIC)
+ return 0;
+
+ break;
+
+ case INSN_CONTEXT_SWITCH:
+ if (func && (!next_insn || !next_insn->hint)) {
+ WARN_FUNC("unsupported instruction in callable function",
+ sec, insn->offset);
+ return 1;
+ }
+ return 0;
+
+ case INSN_STAC:
+ if (state.uaccess) {
+ WARN_FUNC("recursive UACCESS enable", sec, insn->offset);
+ return 1;
+ }
+
+ state.uaccess = true;
+ break;
+
+ case INSN_CLAC:
+ if (!state.uaccess && func) {
+ WARN_FUNC("redundant UACCESS disable", sec, insn->offset);
+ return 1;
+ }
+
+ if (func_uaccess_safe(func) && !state.uaccess_stack) {
+ WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset);
+ return 1;
+ }
+
+ state.uaccess = false;
+ break;
+
+ case INSN_STD:
+ if (state.df) {
+ WARN_FUNC("recursive STD", sec, insn->offset);
+ return 1;
+ }
+
+ state.df = true;
+ break;
+
+ case INSN_CLD:
+ if (!state.df && func) {
+ WARN_FUNC("redundant CLD", sec, insn->offset);
+ return 1;
+ }
+
+ state.df = false;
+ break;
+
+ default:
+ break;
+ }
+
+ if (insn->dead_end)
+ return 0;
+
+ if (!next_insn) {
+ if (state.cfi.cfa.base == CFI_UNDEFINED)
+ return 0;
+ WARN("%s: unexpected end of section", sec->name);
+ return 1;
+ }
+
+ prev_insn = insn;
+ insn = next_insn;
+ }
+
+ return 0;
+}
+
+static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
+{
+ struct instruction *insn;
+ struct insn_state state;
+ int ret, warnings = 0;
+
+ if (!file->hints)
+ return 0;
+
+ init_insn_state(file, &state, sec);
+
+ if (sec) {
+ insn = find_insn(file, sec, 0);
+ if (!insn)
+ return 0;
+ } else {
+ insn = list_first_entry(&file->insn_list, typeof(*insn), list);
+ }
+
+ while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
+ if (insn->hint && !insn->visited && !insn->ignore) {
+ ret = validate_branch(file, insn->func, insn, state);
+ if (ret && opts.backtrace)
+ BT_FUNC("<=== (hint)", insn);
+ warnings += ret;
+ }
+
+ insn = list_next_entry(insn, list);
+ }
+
+ return warnings;
+}
+
+/*
+ * Validate rethunk entry constraint: must untrain RET before the first RET.
+ *
+ * Follow every branch (intra-function) and ensure ANNOTATE_UNRET_END comes
+ * before an actual RET instruction.
+ */
+static int validate_entry(struct objtool_file *file, struct instruction *insn)
+{
+ struct instruction *next, *dest;
+ int ret, warnings = 0;
+
+ for (;;) {
+ next = next_insn_to_validate(file, insn);
+
+ if (insn->visited & VISITED_ENTRY)
+ return 0;
+
+ insn->visited |= VISITED_ENTRY;
+
+ if (!insn->ignore_alts && !list_empty(&insn->alts)) {
+ struct alternative *alt;
+ bool skip_orig = false;
+
+ list_for_each_entry(alt, &insn->alts, list) {
+ if (alt->skip_orig)
+ skip_orig = true;
+
+ ret = validate_entry(file, alt->insn);
+ if (ret) {
+ if (opts.backtrace)
+ BT_FUNC("(alt)", insn);
+ return ret;
+ }
+ }
+
+ if (skip_orig)
+ return 0;
+ }
+
+ switch (insn->type) {
+
+ case INSN_CALL_DYNAMIC:
+ case INSN_JUMP_DYNAMIC:
+ case INSN_JUMP_DYNAMIC_CONDITIONAL:
+ WARN_FUNC("early indirect call", insn->sec, insn->offset);
+ return 1;
+
+ case INSN_JUMP_UNCONDITIONAL:
+ case INSN_JUMP_CONDITIONAL:
+ if (!is_sibling_call(insn)) {
+ if (!insn->jump_dest) {
+ WARN_FUNC("unresolved jump target after linking?!?",
+ insn->sec, insn->offset);
+ return -1;
+ }
+ ret = validate_entry(file, insn->jump_dest);
+ if (ret) {
+ if (opts.backtrace) {
+ BT_FUNC("(branch%s)", insn,
+ insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
+ }
+ return ret;
+ }
+
+ if (insn->type == INSN_JUMP_UNCONDITIONAL)
+ return 0;
+
+ break;
+ }
+
+ /* fallthrough */
+ case INSN_CALL:
+ dest = find_insn(file, insn->call_dest->sec,
+ insn->call_dest->offset);
+ if (!dest) {
+ WARN("Unresolved function after linking!?: %s",
+ insn->call_dest->name);
+ return -1;
+ }
+
+ ret = validate_entry(file, dest);
+ if (ret) {
+ if (opts.backtrace)
+ BT_FUNC("(call)", insn);
+ return ret;
+ }
+ /*
+ * If a call returns without error, it must have seen UNTRAIN_RET.
+ * Therefore any non-error return is a success.
+ */
+ return 0;
+
+ case INSN_RETURN:
+ WARN_FUNC("RET before UNTRAIN", insn->sec, insn->offset);
+ return 1;
+
+ case INSN_NOP:
+ if (insn->retpoline_safe)
+ return 0;
+ break;
+
+ default:
+ break;
+ }
+
+ if (!next) {
+ WARN_FUNC("teh end!", insn->sec, insn->offset);
+ return -1;
+ }
+ insn = next;
+ }
+
+ return warnings;
+}
+
+/*
+ * Validate that all branches starting at 'insn->entry' encounter UNRET_END
+ * before RET.
+ */
+static int validate_unret(struct objtool_file *file)
+{
+ struct instruction *insn;
+ int ret, warnings = 0;
+
+ for_each_insn(file, insn) {
+ if (!insn->entry)
+ continue;
+
+ ret = validate_entry(file, insn);
+ if (ret < 0) {
+ WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset);
+ return ret;
+ }
+ warnings += ret;
+ }
+
+ return warnings;
+}
+
+static int validate_retpoline(struct objtool_file *file)
+{
+ struct instruction *insn;
+ int warnings = 0;
+
+ for_each_insn(file, insn) {
+ if (insn->type != INSN_JUMP_DYNAMIC &&
+ insn->type != INSN_CALL_DYNAMIC &&
+ insn->type != INSN_RETURN)
+ continue;
+
+ if (insn->retpoline_safe)
+ continue;
+
+ /*
+ * .init.text code is ran before userspace and thus doesn't
+ * strictly need retpolines, except for modules which are
+ * loaded late, they very much do need retpoline in their
+ * .init.text
+ */
+ if (!strcmp(insn->sec->name, ".init.text") && !opts.module)
+ continue;
+
+ if (insn->type == INSN_RETURN) {
+ if (opts.rethunk) {
+ WARN_FUNC("'naked' return found in RETHUNK build",
+ insn->sec, insn->offset);
+ } else
+ continue;
+ } else {
+ WARN_FUNC("indirect %s found in RETPOLINE build",
+ insn->sec, insn->offset,
+ insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
+ }
+
+ warnings++;
+ }
+
+ return warnings;
+}
+
+static bool is_kasan_insn(struct instruction *insn)
+{
+ return (insn->type == INSN_CALL &&
+ !strcmp(insn->call_dest->name, "__asan_handle_no_return"));
+}
+
+static bool is_ubsan_insn(struct instruction *insn)
+{
+ return (insn->type == INSN_CALL &&
+ !strcmp(insn->call_dest->name,
+ "__ubsan_handle_builtin_unreachable"));
+}
+
+static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
+{
+ int i;
+ struct instruction *prev_insn;
+
+ if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
+ return true;
+
+ /*
+ * Ignore alternative replacement instructions. This can happen
+ * when a whitelisted function uses one of the ALTERNATIVE macros.
+ */
+ if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
+ !strcmp(insn->sec->name, ".altinstr_aux"))
+ return true;
+
+ /*
+ * Whole archive runs might encounter dead code from weak symbols.
+ * This is where the linker will have dropped the weak symbol in
+ * favour of a regular symbol, but leaves the code in place.
+ *
+ * In this case we'll find a piece of code (whole function) that is not
+ * covered by a !section symbol. Ignore them.
+ */
+ if (opts.link && !insn->func) {
+ int size = find_symbol_hole_containing(insn->sec, insn->offset);
+ unsigned long end = insn->offset + size;
+
+ if (!size) /* not a hole */
+ return false;
+
+ if (size < 0) /* hole until the end */
+ return true;
+
+ sec_for_each_insn_continue(file, insn) {
+ /*
+ * If we reach a visited instruction at or before the
+ * end of the hole, ignore the unreachable.
+ */
+ if (insn->visited)
+ return true;
+
+ if (insn->offset >= end)
+ break;
+
+ /*
+ * If this hole jumps to a .cold function, mark it ignore too.
+ */
+ if (insn->jump_dest && insn->jump_dest->func &&
+ strstr(insn->jump_dest->func->name, ".cold")) {
+ struct instruction *dest = insn->jump_dest;
+ func_for_each_insn(file, dest->func, dest)
+ dest->ignore = true;
+ }
+ }
+
+ return false;
+ }
+
+ if (!insn->func)
+ return false;
+
+ if (insn->func->static_call_tramp)
+ return true;
+
+ /*
+ * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
+ * __builtin_unreachable(). The BUG() macro has an unreachable() after
+ * the UD2, which causes GCC's undefined trap logic to emit another UD2
+ * (or occasionally a JMP to UD2).
+ *
+ * It may also insert a UD2 after calling a __noreturn function.
+ */
+ prev_insn = list_prev_entry(insn, list);
+ if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) &&
+ (insn->type == INSN_BUG ||
+ (insn->type == INSN_JUMP_UNCONDITIONAL &&
+ insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
+ return true;
+
+ /*
+ * Check if this (or a subsequent) instruction is related to
+ * CONFIG_UBSAN or CONFIG_KASAN.
+ *
+ * End the search at 5 instructions to avoid going into the weeds.
+ */
+ for (i = 0; i < 5; i++) {
+
+ if (is_kasan_insn(insn) || is_ubsan_insn(insn))
+ return true;
+
+ if (insn->type == INSN_JUMP_UNCONDITIONAL) {
+ if (insn->jump_dest &&
+ insn->jump_dest->func == insn->func) {
+ insn = insn->jump_dest;
+ continue;
+ }
+
+ break;
+ }
+
+ if (insn->offset + insn->len >= insn->func->offset + insn->func->len)
+ break;
+
+ insn = list_next_entry(insn, list);
+ }
+
+ return false;
+}
+
+static int validate_symbol(struct objtool_file *file, struct section *sec,
+ struct symbol *sym, struct insn_state *state)
+{
+ struct instruction *insn;
+ int ret;
+
+ if (!sym->len) {
+ WARN("%s() is missing an ELF size annotation", sym->name);
+ return 1;
+ }
+
+ if (sym->pfunc != sym || sym->alias != sym)
+ return 0;
+
+ insn = find_insn(file, sec, sym->offset);
+ if (!insn || insn->ignore || insn->visited)
+ return 0;
+
+ state->uaccess = sym->uaccess_safe;
+
+ ret = validate_branch(file, insn->func, insn, *state);
+ if (ret && opts.backtrace)
+ BT_FUNC("<=== (sym)", insn);
+ return ret;
+}
+
+static int validate_section(struct objtool_file *file, struct section *sec)
+{
+ struct insn_state state;
+ struct symbol *func;
+ int warnings = 0;
+
+ list_for_each_entry(func, &sec->symbol_list, list) {
+ if (func->type != STT_FUNC)
+ continue;
+
+ init_insn_state(file, &state, sec);
+ set_func_state(&state.cfi);
+
+ warnings += validate_symbol(file, sec, func, &state);
+ }
+
+ return warnings;
+}
+
+static int validate_noinstr_sections(struct objtool_file *file)
+{
+ struct section *sec;
+ int warnings = 0;
+
+ sec = find_section_by_name(file->elf, ".noinstr.text");
+ if (sec) {
+ warnings += validate_section(file, sec);
+ warnings += validate_unwind_hints(file, sec);
+ }
+
+ sec = find_section_by_name(file->elf, ".entry.text");
+ if (sec) {
+ warnings += validate_section(file, sec);
+ warnings += validate_unwind_hints(file, sec);
+ }
+
+ return warnings;
+}
+
+static int validate_functions(struct objtool_file *file)
+{
+ struct section *sec;
+ int warnings = 0;
+
+ for_each_sec(file, sec) {
+ if (!(sec->sh.sh_flags & SHF_EXECINSTR))
+ continue;
+
+ warnings += validate_section(file, sec);
+ }
+
+ return warnings;
+}
+
+static void mark_endbr_used(struct instruction *insn)
+{
+ if (!list_empty(&insn->call_node))
+ list_del_init(&insn->call_node);
+}
+
+static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
+{
+ struct instruction *dest;
+ struct reloc *reloc;
+ unsigned long off;
+ int warnings = 0;
+
+ /*
+ * Looking for function pointer load relocations. Ignore
+ * direct/indirect branches:
+ */
+ switch (insn->type) {
+ case INSN_CALL:
+ case INSN_CALL_DYNAMIC:
+ case INSN_JUMP_CONDITIONAL:
+ case INSN_JUMP_UNCONDITIONAL:
+ case INSN_JUMP_DYNAMIC:
+ case INSN_JUMP_DYNAMIC_CONDITIONAL:
+ case INSN_RETURN:
+ case INSN_NOP:
+ return 0;
+ default:
+ break;
+ }
+
+ for (reloc = insn_reloc(file, insn);
+ reloc;
+ reloc = find_reloc_by_dest_range(file->elf, insn->sec,
+ reloc->offset + 1,
+ (insn->offset + insn->len) - (reloc->offset + 1))) {
+
+ /*
+ * static_call_update() references the trampoline, which
+ * doesn't have (or need) ENDBR. Skip warning in that case.
+ */
+ if (reloc->sym->static_call_tramp)
+ continue;
+
+ off = reloc->sym->offset;
+ if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32)
+ off += arch_dest_reloc_offset(reloc->addend);
+ else
+ off += reloc->addend;
+
+ dest = find_insn(file, reloc->sym->sec, off);
+ if (!dest)
+ continue;
+
+ if (dest->type == INSN_ENDBR) {
+ mark_endbr_used(dest);
+ continue;
+ }
+
+ if (dest->func && dest->func == insn->func) {
+ /*
+ * Anything from->to self is either _THIS_IP_ or
+ * IRET-to-self.
+ *
+ * There is no sane way to annotate _THIS_IP_ since the
+ * compiler treats the relocation as a constant and is
+ * happy to fold in offsets, skewing any annotation we
+ * do, leading to vast amounts of false-positives.
+ *
+ * There's also compiler generated _THIS_IP_ through
+ * KCOV and such which we have no hope of annotating.
+ *
+ * As such, blanket accept self-references without
+ * issue.
+ */
+ continue;
+ }
+
+ if (dest->noendbr)
+ continue;
+
+ WARN_FUNC("relocation to !ENDBR: %s",
+ insn->sec, insn->offset,
+ offstr(dest->sec, dest->offset));
+
+ warnings++;
+ }
+
+ return warnings;
+}
+
+static int validate_ibt_data_reloc(struct objtool_file *file,
+ struct reloc *reloc)
+{
+ struct instruction *dest;
+
+ dest = find_insn(file, reloc->sym->sec,
+ reloc->sym->offset + reloc->addend);
+ if (!dest)
+ return 0;
+
+ if (dest->type == INSN_ENDBR) {
+ mark_endbr_used(dest);
+ return 0;
+ }
+
+ if (dest->noendbr)
+ return 0;
+
+ WARN_FUNC("data relocation to !ENDBR: %s",
+ reloc->sec->base, reloc->offset,
+ offstr(dest->sec, dest->offset));
+
+ return 1;
+}
+
+/*
+ * Validate IBT rules and remove used ENDBR instructions from the seal list.
+ * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
+ * NOPs) later, in create_ibt_endbr_seal_sections().
+ */
+static int validate_ibt(struct objtool_file *file)
+{
+ struct section *sec;
+ struct reloc *reloc;
+ struct instruction *insn;
+ int warnings = 0;
+
+ for_each_insn(file, insn)
+ warnings += validate_ibt_insn(file, insn);
+
+ for_each_sec(file, sec) {
+
+ /* Already done by validate_ibt_insn() */
+ if (sec->sh.sh_flags & SHF_EXECINSTR)
+ continue;
+
+ if (!sec->reloc)
+ continue;
+
+ /*
+ * These sections can reference text addresses, but not with
+ * the intent to indirect branch to them.
+ */
+ if ((!strncmp(sec->name, ".discard", 8) &&
+ strcmp(sec->name, ".discard.ibt_endbr_noseal")) ||
+ !strncmp(sec->name, ".debug", 6) ||
+ !strcmp(sec->name, ".altinstructions") ||
+ !strcmp(sec->name, ".ibt_endbr_seal") ||
+ !strcmp(sec->name, ".orc_unwind_ip") ||
+ !strcmp(sec->name, ".parainstructions") ||
+ !strcmp(sec->name, ".retpoline_sites") ||
+ !strcmp(sec->name, ".smp_locks") ||
+ !strcmp(sec->name, ".static_call_sites") ||
+ !strcmp(sec->name, "_error_injection_whitelist") ||
+ !strcmp(sec->name, "_kprobe_blacklist") ||
+ !strcmp(sec->name, "__bug_table") ||
+ !strcmp(sec->name, "__ex_table") ||
+ !strcmp(sec->name, "__jump_table") ||
+ !strcmp(sec->name, "__mcount_loc") ||
+ !strcmp(sec->name, ".kcfi_traps") ||
+ strstr(sec->name, "__patchable_function_entries"))
+ continue;
+
+ list_for_each_entry(reloc, &sec->reloc->reloc_list, list)
+ warnings += validate_ibt_data_reloc(file, reloc);
+ }
+
+ return warnings;
+}
+
+static int validate_sls(struct objtool_file *file)
+{
+ struct instruction *insn, *next_insn;
+ int warnings = 0;
+
+ for_each_insn(file, insn) {
+ next_insn = next_insn_same_sec(file, insn);
+
+ if (insn->retpoline_safe)
+ continue;
+
+ switch (insn->type) {
+ case INSN_RETURN:
+ if (!next_insn || next_insn->type != INSN_TRAP) {
+ WARN_FUNC("missing int3 after ret",
+ insn->sec, insn->offset);
+ warnings++;
+ }
+
+ break;
+ case INSN_JUMP_DYNAMIC:
+ if (!next_insn || next_insn->type != INSN_TRAP) {
+ WARN_FUNC("missing int3 after indirect jump",
+ insn->sec, insn->offset);
+ warnings++;
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ return warnings;
+}
+
+static int validate_reachable_instructions(struct objtool_file *file)
+{
+ struct instruction *insn;
+
+ if (file->ignore_unreachables)
+ return 0;
+
+ for_each_insn(file, insn) {
+ if (insn->visited || ignore_unreachable_insn(file, insn))
+ continue;
+
+ WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
+ return 1;
+ }
+
+ return 0;
+}
+
+int check(struct objtool_file *file)
+{
+ int ret, warnings = 0;
+
+ arch_initial_func_cfi_state(&initial_func_cfi);
+ init_cfi_state(&init_cfi);
+ init_cfi_state(&func_cfi);
+ set_func_state(&func_cfi);
+
+ if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
+ goto out;
+
+ cfi_hash_add(&init_cfi);
+ cfi_hash_add(&func_cfi);
+
+ ret = decode_sections(file);
+ if (ret < 0)
+ goto out;
+
+ warnings += ret;
+
+ if (list_empty(&file->insn_list))
+ goto out;
+
+ if (opts.retpoline) {
+ ret = validate_retpoline(file);
+ if (ret < 0)
+ return ret;
+ warnings += ret;
+ }
+
+ if (opts.stackval || opts.orc || opts.uaccess) {
+ ret = validate_functions(file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+
+ ret = validate_unwind_hints(file, NULL);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+
+ if (!warnings) {
+ ret = validate_reachable_instructions(file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+ }
+
+ } else if (opts.noinstr) {
+ ret = validate_noinstr_sections(file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+ }
+
+ if (opts.unret) {
+ /*
+ * Must be after validate_branch() and friends, it plays
+ * further games with insn->visited.
+ */
+ ret = validate_unret(file);
+ if (ret < 0)
+ return ret;
+ warnings += ret;
+ }
+
+ if (opts.ibt) {
+ ret = validate_ibt(file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+ }
+
+ if (opts.sls) {
+ ret = validate_sls(file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+ }
+
+ if (opts.static_call) {
+ ret = create_static_call_sections(file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+ }
+
+ if (opts.retpoline) {
+ ret = create_retpoline_sites_sections(file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+ }
+
+ if (opts.rethunk) {
+ ret = create_return_sites_sections(file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+ }
+
+ if (opts.mcount) {
+ ret = create_mcount_loc_sections(file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+ }
+
+ if (opts.ibt) {
+ ret = create_ibt_endbr_seal_sections(file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+ }
+
+ if (opts.orc && !list_empty(&file->insn_list)) {
+ ret = orc_create(file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+ }
+
+
+ if (opts.stats) {
+ printf("nr_insns_visited: %ld\n", nr_insns_visited);
+ printf("nr_cfi: %ld\n", nr_cfi);
+ printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
+ printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
+ }
+
+out:
+ /*
+ * For now, don't fail the kernel build on fatal warnings. These
+ * errors are still fairly common due to the growing matrix of
+ * supported toolchains and their recent pace of change.
+ */
+ return 0;
+}
diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c
new file mode 100644
index 000000000..7e24b09b1
--- /dev/null
+++ b/tools/objtool/elf.c
@@ -0,0 +1,1362 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * elf.c - ELF access library
+ *
+ * Adapted from kpatch (https://github.com/dynup/kpatch):
+ * Copyright (C) 2013-2015 Josh Poimboeuf <jpoimboe@redhat.com>
+ * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
+ */
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <objtool/builtin.h>
+
+#include <objtool/elf.h>
+#include <objtool/warn.h>
+
+#define MAX_NAME_LEN 128
+
+static inline u32 str_hash(const char *str)
+{
+ return jhash(str, strlen(str), 0);
+}
+
+#define __elf_table(name) (elf->name##_hash)
+#define __elf_bits(name) (elf->name##_bits)
+
+#define elf_hash_add(name, node, key) \
+ hlist_add_head(node, &__elf_table(name)[hash_min(key, __elf_bits(name))])
+
+#define elf_hash_for_each_possible(name, obj, member, key) \
+ hlist_for_each_entry(obj, &__elf_table(name)[hash_min(key, __elf_bits(name))], member)
+
+#define elf_alloc_hash(name, size) \
+({ \
+ __elf_bits(name) = max(10, ilog2(size)); \
+ __elf_table(name) = mmap(NULL, sizeof(struct hlist_head) << __elf_bits(name), \
+ PROT_READ|PROT_WRITE, \
+ MAP_PRIVATE|MAP_ANON, -1, 0); \
+ if (__elf_table(name) == (void *)-1L) { \
+ WARN("mmap fail " #name); \
+ __elf_table(name) = NULL; \
+ } \
+ __elf_table(name); \
+})
+
+static bool symbol_to_offset(struct rb_node *a, const struct rb_node *b)
+{
+ struct symbol *sa = rb_entry(a, struct symbol, node);
+ struct symbol *sb = rb_entry(b, struct symbol, node);
+
+ if (sa->offset < sb->offset)
+ return true;
+ if (sa->offset > sb->offset)
+ return false;
+
+ if (sa->len < sb->len)
+ return true;
+ if (sa->len > sb->len)
+ return false;
+
+ sa->alias = sb;
+
+ return false;
+}
+
+static int symbol_by_offset(const void *key, const struct rb_node *node)
+{
+ const struct symbol *s = rb_entry(node, struct symbol, node);
+ const unsigned long *o = key;
+
+ if (*o < s->offset)
+ return -1;
+ if (*o >= s->offset + s->len)
+ return 1;
+
+ return 0;
+}
+
+struct symbol_hole {
+ unsigned long key;
+ const struct symbol *sym;
+};
+
+/*
+ * Find !section symbol where @offset is after it.
+ */
+static int symbol_hole_by_offset(const void *key, const struct rb_node *node)
+{
+ const struct symbol *s = rb_entry(node, struct symbol, node);
+ struct symbol_hole *sh = (void *)key;
+
+ if (sh->key < s->offset)
+ return -1;
+
+ if (sh->key >= s->offset + s->len) {
+ if (s->type != STT_SECTION)
+ sh->sym = s;
+ return 1;
+ }
+
+ return 0;
+}
+
+struct section *find_section_by_name(const struct elf *elf, const char *name)
+{
+ struct section *sec;
+
+ elf_hash_for_each_possible(section_name, sec, name_hash, str_hash(name)) {
+ if (!strcmp(sec->name, name))
+ return sec;
+ }
+
+ return NULL;
+}
+
+static struct section *find_section_by_index(struct elf *elf,
+ unsigned int idx)
+{
+ struct section *sec;
+
+ elf_hash_for_each_possible(section, sec, hash, idx) {
+ if (sec->idx == idx)
+ return sec;
+ }
+
+ return NULL;
+}
+
+static struct symbol *find_symbol_by_index(struct elf *elf, unsigned int idx)
+{
+ struct symbol *sym;
+
+ elf_hash_for_each_possible(symbol, sym, hash, idx) {
+ if (sym->idx == idx)
+ return sym;
+ }
+
+ return NULL;
+}
+
+struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset)
+{
+ struct rb_node *node;
+
+ rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) {
+ struct symbol *s = rb_entry(node, struct symbol, node);
+
+ if (s->offset == offset && s->type != STT_SECTION)
+ return s;
+ }
+
+ return NULL;
+}
+
+struct symbol *find_func_by_offset(struct section *sec, unsigned long offset)
+{
+ struct rb_node *node;
+
+ rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) {
+ struct symbol *s = rb_entry(node, struct symbol, node);
+
+ if (s->offset == offset && s->type == STT_FUNC)
+ return s;
+ }
+
+ return NULL;
+}
+
+struct symbol *find_symbol_containing(const struct section *sec, unsigned long offset)
+{
+ struct rb_node *node;
+
+ rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) {
+ struct symbol *s = rb_entry(node, struct symbol, node);
+
+ if (s->type != STT_SECTION)
+ return s;
+ }
+
+ return NULL;
+}
+
+/*
+ * Returns size of hole starting at @offset.
+ */
+int find_symbol_hole_containing(const struct section *sec, unsigned long offset)
+{
+ struct symbol_hole hole = {
+ .key = offset,
+ .sym = NULL,
+ };
+ struct rb_node *n;
+ struct symbol *s;
+
+ /*
+ * Find the rightmost symbol for which @offset is after it.
+ */
+ n = rb_find(&hole, &sec->symbol_tree, symbol_hole_by_offset);
+
+ /* found a symbol that contains @offset */
+ if (n)
+ return 0; /* not a hole */
+
+ /* didn't find a symbol for which @offset is after it */
+ if (!hole.sym)
+ return 0; /* not a hole */
+
+ /* @offset >= sym->offset + sym->len, find symbol after it */
+ n = rb_next(&hole.sym->node);
+ if (!n)
+ return -1; /* until end of address space */
+
+ /* hole until start of next symbol */
+ s = rb_entry(n, struct symbol, node);
+ return s->offset - offset;
+}
+
+struct symbol *find_func_containing(struct section *sec, unsigned long offset)
+{
+ struct rb_node *node;
+
+ rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) {
+ struct symbol *s = rb_entry(node, struct symbol, node);
+
+ if (s->type == STT_FUNC)
+ return s;
+ }
+
+ return NULL;
+}
+
+struct symbol *find_symbol_by_name(const struct elf *elf, const char *name)
+{
+ struct symbol *sym;
+
+ elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(name)) {
+ if (!strcmp(sym->name, name))
+ return sym;
+ }
+
+ return NULL;
+}
+
+struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *sec,
+ unsigned long offset, unsigned int len)
+{
+ struct reloc *reloc, *r = NULL;
+ unsigned long o;
+
+ if (!sec->reloc)
+ return NULL;
+
+ sec = sec->reloc;
+
+ for_offset_range(o, offset, offset + len) {
+ elf_hash_for_each_possible(reloc, reloc, hash,
+ sec_offset_hash(sec, o)) {
+ if (reloc->sec != sec)
+ continue;
+
+ if (reloc->offset >= offset && reloc->offset < offset + len) {
+ if (!r || reloc->offset < r->offset)
+ r = reloc;
+ }
+ }
+ if (r)
+ return r;
+ }
+
+ return NULL;
+}
+
+struct reloc *find_reloc_by_dest(const struct elf *elf, struct section *sec, unsigned long offset)
+{
+ return find_reloc_by_dest_range(elf, sec, offset, 1);
+}
+
+static int read_sections(struct elf *elf)
+{
+ Elf_Scn *s = NULL;
+ struct section *sec;
+ size_t shstrndx, sections_nr;
+ int i;
+
+ if (elf_getshdrnum(elf->elf, &sections_nr)) {
+ WARN_ELF("elf_getshdrnum");
+ return -1;
+ }
+
+ if (elf_getshdrstrndx(elf->elf, &shstrndx)) {
+ WARN_ELF("elf_getshdrstrndx");
+ return -1;
+ }
+
+ if (!elf_alloc_hash(section, sections_nr) ||
+ !elf_alloc_hash(section_name, sections_nr))
+ return -1;
+
+ for (i = 0; i < sections_nr; i++) {
+ sec = malloc(sizeof(*sec));
+ if (!sec) {
+ perror("malloc");
+ return -1;
+ }
+ memset(sec, 0, sizeof(*sec));
+
+ INIT_LIST_HEAD(&sec->symbol_list);
+ INIT_LIST_HEAD(&sec->reloc_list);
+
+ s = elf_getscn(elf->elf, i);
+ if (!s) {
+ WARN_ELF("elf_getscn");
+ return -1;
+ }
+
+ sec->idx = elf_ndxscn(s);
+
+ if (!gelf_getshdr(s, &sec->sh)) {
+ WARN_ELF("gelf_getshdr");
+ return -1;
+ }
+
+ sec->name = elf_strptr(elf->elf, shstrndx, sec->sh.sh_name);
+ if (!sec->name) {
+ WARN_ELF("elf_strptr");
+ return -1;
+ }
+
+ if (sec->sh.sh_size != 0) {
+ sec->data = elf_getdata(s, NULL);
+ if (!sec->data) {
+ WARN_ELF("elf_getdata");
+ return -1;
+ }
+ if (sec->data->d_off != 0 ||
+ sec->data->d_size != sec->sh.sh_size) {
+ WARN("unexpected data attributes for %s",
+ sec->name);
+ return -1;
+ }
+ }
+
+ if (sec->sh.sh_flags & SHF_EXECINSTR)
+ elf->text_size += sec->sh.sh_size;
+
+ list_add_tail(&sec->list, &elf->sections);
+ elf_hash_add(section, &sec->hash, sec->idx);
+ elf_hash_add(section_name, &sec->name_hash, str_hash(sec->name));
+ }
+
+ if (opts.stats) {
+ printf("nr_sections: %lu\n", (unsigned long)sections_nr);
+ printf("section_bits: %d\n", elf->section_bits);
+ }
+
+ /* sanity check, one more call to elf_nextscn() should return NULL */
+ if (elf_nextscn(elf->elf, s)) {
+ WARN("section entry mismatch");
+ return -1;
+ }
+
+ return 0;
+}
+
+static void elf_add_symbol(struct elf *elf, struct symbol *sym)
+{
+ struct list_head *entry;
+ struct rb_node *pnode;
+
+ INIT_LIST_HEAD(&sym->pv_target);
+ sym->alias = sym;
+
+ sym->type = GELF_ST_TYPE(sym->sym.st_info);
+ sym->bind = GELF_ST_BIND(sym->sym.st_info);
+
+ if (sym->type == STT_FILE)
+ elf->num_files++;
+
+ sym->offset = sym->sym.st_value;
+ sym->len = sym->sym.st_size;
+
+ rb_add(&sym->node, &sym->sec->symbol_tree, symbol_to_offset);
+ pnode = rb_prev(&sym->node);
+ if (pnode)
+ entry = &rb_entry(pnode, struct symbol, node)->list;
+ else
+ entry = &sym->sec->symbol_list;
+ list_add(&sym->list, entry);
+ elf_hash_add(symbol, &sym->hash, sym->idx);
+ elf_hash_add(symbol_name, &sym->name_hash, str_hash(sym->name));
+
+ /*
+ * Don't store empty STT_NOTYPE symbols in the rbtree. They
+ * can exist within a function, confusing the sorting.
+ */
+ if (!sym->len)
+ rb_erase(&sym->node, &sym->sec->symbol_tree);
+}
+
+static int read_symbols(struct elf *elf)
+{
+ struct section *symtab, *symtab_shndx, *sec;
+ struct symbol *sym, *pfunc;
+ int symbols_nr, i;
+ char *coldstr;
+ Elf_Data *shndx_data = NULL;
+ Elf32_Word shndx;
+
+ symtab = find_section_by_name(elf, ".symtab");
+ if (symtab) {
+ symtab_shndx = find_section_by_name(elf, ".symtab_shndx");
+ if (symtab_shndx)
+ shndx_data = symtab_shndx->data;
+
+ symbols_nr = symtab->sh.sh_size / symtab->sh.sh_entsize;
+ } else {
+ /*
+ * A missing symbol table is actually possible if it's an empty
+ * .o file. This can happen for thunk_64.o. Make sure to at
+ * least allocate the symbol hash tables so we can do symbol
+ * lookups without crashing.
+ */
+ symbols_nr = 0;
+ }
+
+ if (!elf_alloc_hash(symbol, symbols_nr) ||
+ !elf_alloc_hash(symbol_name, symbols_nr))
+ return -1;
+
+ for (i = 0; i < symbols_nr; i++) {
+ sym = malloc(sizeof(*sym));
+ if (!sym) {
+ perror("malloc");
+ return -1;
+ }
+ memset(sym, 0, sizeof(*sym));
+
+ sym->idx = i;
+
+ if (!gelf_getsymshndx(symtab->data, shndx_data, i, &sym->sym,
+ &shndx)) {
+ WARN_ELF("gelf_getsymshndx");
+ goto err;
+ }
+
+ sym->name = elf_strptr(elf->elf, symtab->sh.sh_link,
+ sym->sym.st_name);
+ if (!sym->name) {
+ WARN_ELF("elf_strptr");
+ goto err;
+ }
+
+ if ((sym->sym.st_shndx > SHN_UNDEF &&
+ sym->sym.st_shndx < SHN_LORESERVE) ||
+ (shndx_data && sym->sym.st_shndx == SHN_XINDEX)) {
+ if (sym->sym.st_shndx != SHN_XINDEX)
+ shndx = sym->sym.st_shndx;
+
+ sym->sec = find_section_by_index(elf, shndx);
+ if (!sym->sec) {
+ WARN("couldn't find section for symbol %s",
+ sym->name);
+ goto err;
+ }
+ if (GELF_ST_TYPE(sym->sym.st_info) == STT_SECTION) {
+ sym->name = sym->sec->name;
+ sym->sec->sym = sym;
+ }
+ } else
+ sym->sec = find_section_by_index(elf, 0);
+
+ elf_add_symbol(elf, sym);
+ }
+
+ if (opts.stats) {
+ printf("nr_symbols: %lu\n", (unsigned long)symbols_nr);
+ printf("symbol_bits: %d\n", elf->symbol_bits);
+ }
+
+ /* Create parent/child links for any cold subfunctions */
+ list_for_each_entry(sec, &elf->sections, list) {
+ list_for_each_entry(sym, &sec->symbol_list, list) {
+ char pname[MAX_NAME_LEN + 1];
+ size_t pnamelen;
+ if (sym->type != STT_FUNC)
+ continue;
+
+ if (sym->pfunc == NULL)
+ sym->pfunc = sym;
+
+ if (sym->cfunc == NULL)
+ sym->cfunc = sym;
+
+ coldstr = strstr(sym->name, ".cold");
+ if (!coldstr)
+ continue;
+
+ pnamelen = coldstr - sym->name;
+ if (pnamelen > MAX_NAME_LEN) {
+ WARN("%s(): parent function name exceeds maximum length of %d characters",
+ sym->name, MAX_NAME_LEN);
+ return -1;
+ }
+
+ strncpy(pname, sym->name, pnamelen);
+ pname[pnamelen] = '\0';
+ pfunc = find_symbol_by_name(elf, pname);
+
+ if (!pfunc) {
+ WARN("%s(): can't find parent function",
+ sym->name);
+ return -1;
+ }
+
+ sym->pfunc = pfunc;
+ pfunc->cfunc = sym;
+
+ /*
+ * Unfortunately, -fnoreorder-functions puts the child
+ * inside the parent. Remove the overlap so we can
+ * have sane assumptions.
+ *
+ * Note that pfunc->len now no longer matches
+ * pfunc->sym.st_size.
+ */
+ if (sym->sec == pfunc->sec &&
+ sym->offset >= pfunc->offset &&
+ sym->offset + sym->len == pfunc->offset + pfunc->len) {
+ pfunc->len -= sym->len;
+ }
+ }
+ }
+
+ return 0;
+
+err:
+ free(sym);
+ return -1;
+}
+
+static struct section *elf_create_reloc_section(struct elf *elf,
+ struct section *base,
+ int reltype);
+
+int elf_add_reloc(struct elf *elf, struct section *sec, unsigned long offset,
+ unsigned int type, struct symbol *sym, s64 addend)
+{
+ struct reloc *reloc;
+
+ if (!sec->reloc && !elf_create_reloc_section(elf, sec, SHT_RELA))
+ return -1;
+
+ reloc = malloc(sizeof(*reloc));
+ if (!reloc) {
+ perror("malloc");
+ return -1;
+ }
+ memset(reloc, 0, sizeof(*reloc));
+
+ reloc->sec = sec->reloc;
+ reloc->offset = offset;
+ reloc->type = type;
+ reloc->sym = sym;
+ reloc->addend = addend;
+
+ list_add_tail(&reloc->list, &sec->reloc->reloc_list);
+ elf_hash_add(reloc, &reloc->hash, reloc_hash(reloc));
+
+ sec->reloc->sh.sh_size += sec->reloc->sh.sh_entsize;
+ sec->reloc->changed = true;
+
+ return 0;
+}
+
+/*
+ * Ensure that any reloc section containing references to @sym is marked
+ * changed such that it will get re-generated in elf_rebuild_reloc_sections()
+ * with the new symbol index.
+ */
+static void elf_dirty_reloc_sym(struct elf *elf, struct symbol *sym)
+{
+ struct section *sec;
+
+ list_for_each_entry(sec, &elf->sections, list) {
+ struct reloc *reloc;
+
+ if (sec->changed)
+ continue;
+
+ list_for_each_entry(reloc, &sec->reloc_list, list) {
+ if (reloc->sym == sym) {
+ sec->changed = true;
+ break;
+ }
+ }
+ }
+}
+
+/*
+ * The libelf API is terrible; gelf_update_sym*() takes a data block relative
+ * index value, *NOT* the symbol index. As such, iterate the data blocks and
+ * adjust index until it fits.
+ *
+ * If no data block is found, allow adding a new data block provided the index
+ * is only one past the end.
+ */
+static int elf_update_symbol(struct elf *elf, struct section *symtab,
+ struct section *symtab_shndx, struct symbol *sym)
+{
+ Elf32_Word shndx = sym->sec ? sym->sec->idx : SHN_UNDEF;
+ Elf_Data *symtab_data = NULL, *shndx_data = NULL;
+ Elf64_Xword entsize = symtab->sh.sh_entsize;
+ int max_idx, idx = sym->idx;
+ Elf_Scn *s, *t = NULL;
+ bool is_special_shndx = sym->sym.st_shndx >= SHN_LORESERVE &&
+ sym->sym.st_shndx != SHN_XINDEX;
+
+ if (is_special_shndx)
+ shndx = sym->sym.st_shndx;
+
+ s = elf_getscn(elf->elf, symtab->idx);
+ if (!s) {
+ WARN_ELF("elf_getscn");
+ return -1;
+ }
+
+ if (symtab_shndx) {
+ t = elf_getscn(elf->elf, symtab_shndx->idx);
+ if (!t) {
+ WARN_ELF("elf_getscn");
+ return -1;
+ }
+ }
+
+ for (;;) {
+ /* get next data descriptor for the relevant sections */
+ symtab_data = elf_getdata(s, symtab_data);
+ if (t)
+ shndx_data = elf_getdata(t, shndx_data);
+
+ /* end-of-list */
+ if (!symtab_data) {
+ void *buf;
+
+ if (idx) {
+ /* we don't do holes in symbol tables */
+ WARN("index out of range");
+ return -1;
+ }
+
+ /* if @idx == 0, it's the next contiguous entry, create it */
+ symtab_data = elf_newdata(s);
+ if (t)
+ shndx_data = elf_newdata(t);
+
+ buf = calloc(1, entsize);
+ if (!buf) {
+ WARN("malloc");
+ return -1;
+ }
+
+ symtab_data->d_buf = buf;
+ symtab_data->d_size = entsize;
+ symtab_data->d_align = 1;
+ symtab_data->d_type = ELF_T_SYM;
+
+ symtab->sh.sh_size += entsize;
+ symtab->changed = true;
+
+ if (t) {
+ shndx_data->d_buf = &sym->sec->idx;
+ shndx_data->d_size = sizeof(Elf32_Word);
+ shndx_data->d_align = sizeof(Elf32_Word);
+ shndx_data->d_type = ELF_T_WORD;
+
+ symtab_shndx->sh.sh_size += sizeof(Elf32_Word);
+ symtab_shndx->changed = true;
+ }
+
+ break;
+ }
+
+ /* empty blocks should not happen */
+ if (!symtab_data->d_size) {
+ WARN("zero size data");
+ return -1;
+ }
+
+ /* is this the right block? */
+ max_idx = symtab_data->d_size / entsize;
+ if (idx < max_idx)
+ break;
+
+ /* adjust index and try again */
+ idx -= max_idx;
+ }
+
+ /* something went side-ways */
+ if (idx < 0) {
+ WARN("negative index");
+ return -1;
+ }
+
+ /* setup extended section index magic and write the symbol */
+ if ((shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) || is_special_shndx) {
+ sym->sym.st_shndx = shndx;
+ if (!shndx_data)
+ shndx = 0;
+ } else {
+ sym->sym.st_shndx = SHN_XINDEX;
+ if (!shndx_data) {
+ WARN("no .symtab_shndx");
+ return -1;
+ }
+ }
+
+ if (!gelf_update_symshndx(symtab_data, shndx_data, idx, &sym->sym, shndx)) {
+ WARN_ELF("gelf_update_symshndx");
+ return -1;
+ }
+
+ return 0;
+}
+
+static struct symbol *
+elf_create_section_symbol(struct elf *elf, struct section *sec)
+{
+ struct section *symtab, *symtab_shndx;
+ Elf32_Word first_non_local, new_idx;
+ struct symbol *sym, *old;
+
+ symtab = find_section_by_name(elf, ".symtab");
+ if (symtab) {
+ symtab_shndx = find_section_by_name(elf, ".symtab_shndx");
+ } else {
+ WARN("no .symtab");
+ return NULL;
+ }
+
+ sym = calloc(1, sizeof(*sym));
+ if (!sym) {
+ perror("malloc");
+ return NULL;
+ }
+
+ sym->name = sec->name;
+ sym->sec = sec;
+
+ // st_name 0
+ sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION);
+ // st_other 0
+ // st_value 0
+ // st_size 0
+
+ /*
+ * Move the first global symbol, as per sh_info, into a new, higher
+ * symbol index. This fees up a spot for a new local symbol.
+ */
+ first_non_local = symtab->sh.sh_info;
+ new_idx = symtab->sh.sh_size / symtab->sh.sh_entsize;
+ old = find_symbol_by_index(elf, first_non_local);
+ if (old) {
+ old->idx = new_idx;
+
+ hlist_del(&old->hash);
+ elf_hash_add(symbol, &old->hash, old->idx);
+
+ elf_dirty_reloc_sym(elf, old);
+
+ if (elf_update_symbol(elf, symtab, symtab_shndx, old)) {
+ WARN("elf_update_symbol move");
+ return NULL;
+ }
+
+ new_idx = first_non_local;
+ }
+
+ sym->idx = new_idx;
+ if (elf_update_symbol(elf, symtab, symtab_shndx, sym)) {
+ WARN("elf_update_symbol");
+ return NULL;
+ }
+
+ /*
+ * Either way, we added a LOCAL symbol.
+ */
+ symtab->sh.sh_info += 1;
+
+ elf_add_symbol(elf, sym);
+
+ return sym;
+}
+
+int elf_add_reloc_to_insn(struct elf *elf, struct section *sec,
+ unsigned long offset, unsigned int type,
+ struct section *insn_sec, unsigned long insn_off)
+{
+ struct symbol *sym = insn_sec->sym;
+ int addend = insn_off;
+
+ if (!sym) {
+ /*
+ * Due to how weak functions work, we must use section based
+ * relocations. Symbol based relocations would result in the
+ * weak and non-weak function annotations being overlaid on the
+ * non-weak function after linking.
+ */
+ sym = elf_create_section_symbol(elf, insn_sec);
+ if (!sym)
+ return -1;
+
+ insn_sec->sym = sym;
+ }
+
+ return elf_add_reloc(elf, sec, offset, type, sym, addend);
+}
+
+static int read_rel_reloc(struct section *sec, int i, struct reloc *reloc, unsigned int *symndx)
+{
+ if (!gelf_getrel(sec->data, i, &reloc->rel)) {
+ WARN_ELF("gelf_getrel");
+ return -1;
+ }
+ reloc->type = GELF_R_TYPE(reloc->rel.r_info);
+ reloc->addend = 0;
+ reloc->offset = reloc->rel.r_offset;
+ *symndx = GELF_R_SYM(reloc->rel.r_info);
+ return 0;
+}
+
+static int read_rela_reloc(struct section *sec, int i, struct reloc *reloc, unsigned int *symndx)
+{
+ if (!gelf_getrela(sec->data, i, &reloc->rela)) {
+ WARN_ELF("gelf_getrela");
+ return -1;
+ }
+ reloc->type = GELF_R_TYPE(reloc->rela.r_info);
+ reloc->addend = reloc->rela.r_addend;
+ reloc->offset = reloc->rela.r_offset;
+ *symndx = GELF_R_SYM(reloc->rela.r_info);
+ return 0;
+}
+
+static int read_relocs(struct elf *elf)
+{
+ struct section *sec;
+ struct reloc *reloc;
+ int i;
+ unsigned int symndx;
+ unsigned long nr_reloc, max_reloc = 0, tot_reloc = 0;
+
+ if (!elf_alloc_hash(reloc, elf->text_size / 16))
+ return -1;
+
+ list_for_each_entry(sec, &elf->sections, list) {
+ if ((sec->sh.sh_type != SHT_RELA) &&
+ (sec->sh.sh_type != SHT_REL))
+ continue;
+
+ sec->base = find_section_by_index(elf, sec->sh.sh_info);
+ if (!sec->base) {
+ WARN("can't find base section for reloc section %s",
+ sec->name);
+ return -1;
+ }
+
+ sec->base->reloc = sec;
+
+ nr_reloc = 0;
+ for (i = 0; i < sec->sh.sh_size / sec->sh.sh_entsize; i++) {
+ reloc = malloc(sizeof(*reloc));
+ if (!reloc) {
+ perror("malloc");
+ return -1;
+ }
+ memset(reloc, 0, sizeof(*reloc));
+ switch (sec->sh.sh_type) {
+ case SHT_REL:
+ if (read_rel_reloc(sec, i, reloc, &symndx))
+ return -1;
+ break;
+ case SHT_RELA:
+ if (read_rela_reloc(sec, i, reloc, &symndx))
+ return -1;
+ break;
+ default: return -1;
+ }
+
+ reloc->sec = sec;
+ reloc->idx = i;
+ reloc->sym = find_symbol_by_index(elf, symndx);
+ if (!reloc->sym) {
+ WARN("can't find reloc entry symbol %d for %s",
+ symndx, sec->name);
+ return -1;
+ }
+
+ list_add_tail(&reloc->list, &sec->reloc_list);
+ elf_hash_add(reloc, &reloc->hash, reloc_hash(reloc));
+
+ nr_reloc++;
+ }
+ max_reloc = max(max_reloc, nr_reloc);
+ tot_reloc += nr_reloc;
+ }
+
+ if (opts.stats) {
+ printf("max_reloc: %lu\n", max_reloc);
+ printf("tot_reloc: %lu\n", tot_reloc);
+ printf("reloc_bits: %d\n", elf->reloc_bits);
+ }
+
+ return 0;
+}
+
+struct elf *elf_open_read(const char *name, int flags)
+{
+ struct elf *elf;
+ Elf_Cmd cmd;
+
+ elf_version(EV_CURRENT);
+
+ elf = malloc(sizeof(*elf));
+ if (!elf) {
+ perror("malloc");
+ return NULL;
+ }
+ memset(elf, 0, offsetof(struct elf, sections));
+
+ INIT_LIST_HEAD(&elf->sections);
+
+ elf->fd = open(name, flags);
+ if (elf->fd == -1) {
+ fprintf(stderr, "objtool: Can't open '%s': %s\n",
+ name, strerror(errno));
+ goto err;
+ }
+
+ if ((flags & O_ACCMODE) == O_RDONLY)
+ cmd = ELF_C_READ_MMAP;
+ else if ((flags & O_ACCMODE) == O_RDWR)
+ cmd = ELF_C_RDWR;
+ else /* O_WRONLY */
+ cmd = ELF_C_WRITE;
+
+ elf->elf = elf_begin(elf->fd, cmd, NULL);
+ if (!elf->elf) {
+ WARN_ELF("elf_begin");
+ goto err;
+ }
+
+ if (!gelf_getehdr(elf->elf, &elf->ehdr)) {
+ WARN_ELF("gelf_getehdr");
+ goto err;
+ }
+
+ if (read_sections(elf))
+ goto err;
+
+ if (read_symbols(elf))
+ goto err;
+
+ if (read_relocs(elf))
+ goto err;
+
+ return elf;
+
+err:
+ elf_close(elf);
+ return NULL;
+}
+
+static int elf_add_string(struct elf *elf, struct section *strtab, char *str)
+{
+ Elf_Data *data;
+ Elf_Scn *s;
+ int len;
+
+ if (!strtab)
+ strtab = find_section_by_name(elf, ".strtab");
+ if (!strtab) {
+ WARN("can't find .strtab section");
+ return -1;
+ }
+
+ s = elf_getscn(elf->elf, strtab->idx);
+ if (!s) {
+ WARN_ELF("elf_getscn");
+ return -1;
+ }
+
+ data = elf_newdata(s);
+ if (!data) {
+ WARN_ELF("elf_newdata");
+ return -1;
+ }
+
+ data->d_buf = str;
+ data->d_size = strlen(str) + 1;
+ data->d_align = 1;
+
+ len = strtab->sh.sh_size;
+ strtab->sh.sh_size += data->d_size;
+ strtab->changed = true;
+
+ return len;
+}
+
+struct section *elf_create_section(struct elf *elf, const char *name,
+ unsigned int sh_flags, size_t entsize, int nr)
+{
+ struct section *sec, *shstrtab;
+ size_t size = entsize * nr;
+ Elf_Scn *s;
+
+ sec = malloc(sizeof(*sec));
+ if (!sec) {
+ perror("malloc");
+ return NULL;
+ }
+ memset(sec, 0, sizeof(*sec));
+
+ INIT_LIST_HEAD(&sec->symbol_list);
+ INIT_LIST_HEAD(&sec->reloc_list);
+
+ s = elf_newscn(elf->elf);
+ if (!s) {
+ WARN_ELF("elf_newscn");
+ return NULL;
+ }
+
+ sec->name = strdup(name);
+ if (!sec->name) {
+ perror("strdup");
+ return NULL;
+ }
+
+ sec->idx = elf_ndxscn(s);
+ sec->changed = true;
+
+ sec->data = elf_newdata(s);
+ if (!sec->data) {
+ WARN_ELF("elf_newdata");
+ return NULL;
+ }
+
+ sec->data->d_size = size;
+ sec->data->d_align = 1;
+
+ if (size) {
+ sec->data->d_buf = malloc(size);
+ if (!sec->data->d_buf) {
+ perror("malloc");
+ return NULL;
+ }
+ memset(sec->data->d_buf, 0, size);
+ }
+
+ if (!gelf_getshdr(s, &sec->sh)) {
+ WARN_ELF("gelf_getshdr");
+ return NULL;
+ }
+
+ sec->sh.sh_size = size;
+ sec->sh.sh_entsize = entsize;
+ sec->sh.sh_type = SHT_PROGBITS;
+ sec->sh.sh_addralign = 1;
+ sec->sh.sh_flags = SHF_ALLOC | sh_flags;
+
+ /* Add section name to .shstrtab (or .strtab for Clang) */
+ shstrtab = find_section_by_name(elf, ".shstrtab");
+ if (!shstrtab)
+ shstrtab = find_section_by_name(elf, ".strtab");
+ if (!shstrtab) {
+ WARN("can't find .shstrtab or .strtab section");
+ return NULL;
+ }
+ sec->sh.sh_name = elf_add_string(elf, shstrtab, sec->name);
+ if (sec->sh.sh_name == -1)
+ return NULL;
+
+ list_add_tail(&sec->list, &elf->sections);
+ elf_hash_add(section, &sec->hash, sec->idx);
+ elf_hash_add(section_name, &sec->name_hash, str_hash(sec->name));
+
+ elf->changed = true;
+
+ return sec;
+}
+
+static struct section *elf_create_rel_reloc_section(struct elf *elf, struct section *base)
+{
+ char *relocname;
+ struct section *sec;
+
+ relocname = malloc(strlen(base->name) + strlen(".rel") + 1);
+ if (!relocname) {
+ perror("malloc");
+ return NULL;
+ }
+ strcpy(relocname, ".rel");
+ strcat(relocname, base->name);
+
+ sec = elf_create_section(elf, relocname, 0, sizeof(GElf_Rel), 0);
+ free(relocname);
+ if (!sec)
+ return NULL;
+
+ base->reloc = sec;
+ sec->base = base;
+
+ sec->sh.sh_type = SHT_REL;
+ sec->sh.sh_addralign = 8;
+ sec->sh.sh_link = find_section_by_name(elf, ".symtab")->idx;
+ sec->sh.sh_info = base->idx;
+ sec->sh.sh_flags = SHF_INFO_LINK;
+
+ return sec;
+}
+
+static struct section *elf_create_rela_reloc_section(struct elf *elf, struct section *base)
+{
+ char *relocname;
+ struct section *sec;
+
+ relocname = malloc(strlen(base->name) + strlen(".rela") + 1);
+ if (!relocname) {
+ perror("malloc");
+ return NULL;
+ }
+ strcpy(relocname, ".rela");
+ strcat(relocname, base->name);
+
+ sec = elf_create_section(elf, relocname, 0, sizeof(GElf_Rela), 0);
+ free(relocname);
+ if (!sec)
+ return NULL;
+
+ base->reloc = sec;
+ sec->base = base;
+
+ sec->sh.sh_type = SHT_RELA;
+ sec->sh.sh_addralign = 8;
+ sec->sh.sh_link = find_section_by_name(elf, ".symtab")->idx;
+ sec->sh.sh_info = base->idx;
+ sec->sh.sh_flags = SHF_INFO_LINK;
+
+ return sec;
+}
+
+static struct section *elf_create_reloc_section(struct elf *elf,
+ struct section *base,
+ int reltype)
+{
+ switch (reltype) {
+ case SHT_REL: return elf_create_rel_reloc_section(elf, base);
+ case SHT_RELA: return elf_create_rela_reloc_section(elf, base);
+ default: return NULL;
+ }
+}
+
+static int elf_rebuild_rel_reloc_section(struct section *sec)
+{
+ struct reloc *reloc;
+ int idx = 0;
+ void *buf;
+
+ /* Allocate a buffer for relocations */
+ buf = malloc(sec->sh.sh_size);
+ if (!buf) {
+ perror("malloc");
+ return -1;
+ }
+
+ sec->data->d_buf = buf;
+ sec->data->d_size = sec->sh.sh_size;
+ sec->data->d_type = ELF_T_REL;
+
+ idx = 0;
+ list_for_each_entry(reloc, &sec->reloc_list, list) {
+ reloc->rel.r_offset = reloc->offset;
+ reloc->rel.r_info = GELF_R_INFO(reloc->sym->idx, reloc->type);
+ if (!gelf_update_rel(sec->data, idx, &reloc->rel)) {
+ WARN_ELF("gelf_update_rel");
+ return -1;
+ }
+ idx++;
+ }
+
+ return 0;
+}
+
+static int elf_rebuild_rela_reloc_section(struct section *sec)
+{
+ struct reloc *reloc;
+ int idx = 0;
+ void *buf;
+
+ /* Allocate a buffer for relocations with addends */
+ buf = malloc(sec->sh.sh_size);
+ if (!buf) {
+ perror("malloc");
+ return -1;
+ }
+
+ sec->data->d_buf = buf;
+ sec->data->d_size = sec->sh.sh_size;
+ sec->data->d_type = ELF_T_RELA;
+
+ idx = 0;
+ list_for_each_entry(reloc, &sec->reloc_list, list) {
+ reloc->rela.r_offset = reloc->offset;
+ reloc->rela.r_addend = reloc->addend;
+ reloc->rela.r_info = GELF_R_INFO(reloc->sym->idx, reloc->type);
+ if (!gelf_update_rela(sec->data, idx, &reloc->rela)) {
+ WARN_ELF("gelf_update_rela");
+ return -1;
+ }
+ idx++;
+ }
+
+ return 0;
+}
+
+static int elf_rebuild_reloc_section(struct elf *elf, struct section *sec)
+{
+ switch (sec->sh.sh_type) {
+ case SHT_REL: return elf_rebuild_rel_reloc_section(sec);
+ case SHT_RELA: return elf_rebuild_rela_reloc_section(sec);
+ default: return -1;
+ }
+}
+
+int elf_write_insn(struct elf *elf, struct section *sec,
+ unsigned long offset, unsigned int len,
+ const char *insn)
+{
+ Elf_Data *data = sec->data;
+
+ if (data->d_type != ELF_T_BYTE || data->d_off) {
+ WARN("write to unexpected data for section: %s", sec->name);
+ return -1;
+ }
+
+ memcpy(data->d_buf + offset, insn, len);
+ elf_flagdata(data, ELF_C_SET, ELF_F_DIRTY);
+
+ elf->changed = true;
+
+ return 0;
+}
+
+int elf_write_reloc(struct elf *elf, struct reloc *reloc)
+{
+ struct section *sec = reloc->sec;
+
+ if (sec->sh.sh_type == SHT_REL) {
+ reloc->rel.r_info = GELF_R_INFO(reloc->sym->idx, reloc->type);
+ reloc->rel.r_offset = reloc->offset;
+
+ if (!gelf_update_rel(sec->data, reloc->idx, &reloc->rel)) {
+ WARN_ELF("gelf_update_rel");
+ return -1;
+ }
+ } else {
+ reloc->rela.r_info = GELF_R_INFO(reloc->sym->idx, reloc->type);
+ reloc->rela.r_addend = reloc->addend;
+ reloc->rela.r_offset = reloc->offset;
+
+ if (!gelf_update_rela(sec->data, reloc->idx, &reloc->rela)) {
+ WARN_ELF("gelf_update_rela");
+ return -1;
+ }
+ }
+
+ elf->changed = true;
+
+ return 0;
+}
+
+int elf_write(struct elf *elf)
+{
+ struct section *sec;
+ Elf_Scn *s;
+
+ if (opts.dryrun)
+ return 0;
+
+ /* Update changed relocation sections and section headers: */
+ list_for_each_entry(sec, &elf->sections, list) {
+ if (sec->changed) {
+ s = elf_getscn(elf->elf, sec->idx);
+ if (!s) {
+ WARN_ELF("elf_getscn");
+ return -1;
+ }
+ if (!gelf_update_shdr(s, &sec->sh)) {
+ WARN_ELF("gelf_update_shdr");
+ return -1;
+ }
+
+ if (sec->base &&
+ elf_rebuild_reloc_section(elf, sec)) {
+ WARN("elf_rebuild_reloc_section");
+ return -1;
+ }
+
+ sec->changed = false;
+ elf->changed = true;
+ }
+ }
+
+ /* Make sure the new section header entries get updated properly. */
+ elf_flagelf(elf->elf, ELF_C_SET, ELF_F_DIRTY);
+
+ /* Write all changes to the file. */
+ if (elf_update(elf->elf, ELF_C_WRITE) < 0) {
+ WARN_ELF("elf_update");
+ return -1;
+ }
+
+ elf->changed = false;
+
+ return 0;
+}
+
+void elf_close(struct elf *elf)
+{
+ struct section *sec, *tmpsec;
+ struct symbol *sym, *tmpsym;
+ struct reloc *reloc, *tmpreloc;
+
+ if (elf->elf)
+ elf_end(elf->elf);
+
+ if (elf->fd > 0)
+ close(elf->fd);
+
+ list_for_each_entry_safe(sec, tmpsec, &elf->sections, list) {
+ list_for_each_entry_safe(sym, tmpsym, &sec->symbol_list, list) {
+ list_del(&sym->list);
+ hash_del(&sym->hash);
+ free(sym);
+ }
+ list_for_each_entry_safe(reloc, tmpreloc, &sec->reloc_list, list) {
+ list_del(&reloc->list);
+ hash_del(&reloc->hash);
+ free(reloc);
+ }
+ list_del(&sec->list);
+ free(sec);
+ }
+
+ free(elf);
+}
diff --git a/tools/objtool/include/objtool/arch.h b/tools/objtool/include/objtool/arch.h
new file mode 100644
index 000000000..861c0c60a
--- /dev/null
+++ b/tools/objtool/include/objtool/arch.h
@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com>
+ */
+
+#ifndef _ARCH_H
+#define _ARCH_H
+
+#include <stdbool.h>
+#include <linux/list.h>
+#include <objtool/objtool.h>
+#include <objtool/cfi.h>
+
+enum insn_type {
+ INSN_JUMP_CONDITIONAL,
+ INSN_JUMP_UNCONDITIONAL,
+ INSN_JUMP_DYNAMIC,
+ INSN_JUMP_DYNAMIC_CONDITIONAL,
+ INSN_CALL,
+ INSN_CALL_DYNAMIC,
+ INSN_RETURN,
+ INSN_CONTEXT_SWITCH,
+ INSN_BUG,
+ INSN_NOP,
+ INSN_STAC,
+ INSN_CLAC,
+ INSN_STD,
+ INSN_CLD,
+ INSN_TRAP,
+ INSN_ENDBR,
+ INSN_OTHER,
+};
+
+enum op_dest_type {
+ OP_DEST_REG,
+ OP_DEST_REG_INDIRECT,
+ OP_DEST_MEM,
+ OP_DEST_PUSH,
+ OP_DEST_PUSHF,
+};
+
+struct op_dest {
+ enum op_dest_type type;
+ unsigned char reg;
+ int offset;
+};
+
+enum op_src_type {
+ OP_SRC_REG,
+ OP_SRC_REG_INDIRECT,
+ OP_SRC_CONST,
+ OP_SRC_POP,
+ OP_SRC_POPF,
+ OP_SRC_ADD,
+ OP_SRC_AND,
+};
+
+struct op_src {
+ enum op_src_type type;
+ unsigned char reg;
+ int offset;
+};
+
+struct stack_op {
+ struct op_dest dest;
+ struct op_src src;
+ struct list_head list;
+};
+
+struct instruction;
+
+void arch_initial_func_cfi_state(struct cfi_init_state *state);
+
+int arch_decode_instruction(struct objtool_file *file, const struct section *sec,
+ unsigned long offset, unsigned int maxlen,
+ unsigned int *len, enum insn_type *type,
+ unsigned long *immediate,
+ struct list_head *ops_list);
+
+bool arch_callee_saved_reg(unsigned char reg);
+
+unsigned long arch_jump_destination(struct instruction *insn);
+
+unsigned long arch_dest_reloc_offset(int addend);
+
+const char *arch_nop_insn(int len);
+const char *arch_ret_insn(int len);
+
+int arch_decode_hint_reg(u8 sp_reg, int *base);
+
+bool arch_is_retpoline(struct symbol *sym);
+bool arch_is_rethunk(struct symbol *sym);
+bool arch_is_embedded_insn(struct symbol *sym);
+
+int arch_rewrite_retpolines(struct objtool_file *file);
+
+#endif /* _ARCH_H */
diff --git a/tools/objtool/include/objtool/builtin.h b/tools/objtool/include/objtool/builtin.h
new file mode 100644
index 000000000..42a52f1a0
--- /dev/null
+++ b/tools/objtool/include/objtool/builtin.h
@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com>
+ */
+#ifndef _BUILTIN_H
+#define _BUILTIN_H
+
+#include <subcmd/parse-options.h>
+
+extern const struct option check_options[];
+
+struct opts {
+ /* actions: */
+ bool dump_orc;
+ bool hack_jump_label;
+ bool hack_noinstr;
+ bool ibt;
+ bool mcount;
+ bool noinstr;
+ bool orc;
+ bool retpoline;
+ bool rethunk;
+ bool unret;
+ bool sls;
+ bool stackval;
+ bool static_call;
+ bool uaccess;
+
+ /* options: */
+ bool backtrace;
+ bool backup;
+ bool dryrun;
+ bool link;
+ bool module;
+ bool no_unreachable;
+ bool sec_address;
+ bool stats;
+};
+
+extern struct opts opts;
+
+extern int cmd_parse_options(int argc, const char **argv, const char * const usage[]);
+
+extern int objtool_run(int argc, const char **argv);
+
+#endif /* _BUILTIN_H */
diff --git a/tools/objtool/include/objtool/cfi.h b/tools/objtool/include/objtool/cfi.h
new file mode 100644
index 000000000..f11d1ac1d
--- /dev/null
+++ b/tools/objtool/include/objtool/cfi.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
+ */
+
+#ifndef _OBJTOOL_CFI_H
+#define _OBJTOOL_CFI_H
+
+#include <arch/cfi_regs.h>
+#include <linux/list.h>
+
+#define CFI_UNDEFINED -1
+#define CFI_CFA -2
+#define CFI_SP_INDIRECT -3
+#define CFI_BP_INDIRECT -4
+
+struct cfi_reg {
+ int base;
+ int offset;
+};
+
+struct cfi_init_state {
+ struct cfi_reg regs[CFI_NUM_REGS];
+ struct cfi_reg cfa;
+};
+
+struct cfi_state {
+ struct hlist_node hash; /* must be first, cficmp() */
+ struct cfi_reg regs[CFI_NUM_REGS];
+ struct cfi_reg vals[CFI_NUM_REGS];
+ struct cfi_reg cfa;
+ int stack_size;
+ int drap_reg, drap_offset;
+ unsigned char type;
+ bool bp_scratch;
+ bool drap;
+ bool end;
+};
+
+#endif /* _OBJTOOL_CFI_H */
diff --git a/tools/objtool/include/objtool/check.h b/tools/objtool/include/objtool/check.h
new file mode 100644
index 000000000..036129ceb
--- /dev/null
+++ b/tools/objtool/include/objtool/check.h
@@ -0,0 +1,109 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
+ */
+
+#ifndef _CHECK_H
+#define _CHECK_H
+
+#include <stdbool.h>
+#include <objtool/cfi.h>
+#include <objtool/arch.h>
+
+struct insn_state {
+ struct cfi_state cfi;
+ unsigned int uaccess_stack;
+ bool uaccess;
+ bool df;
+ bool noinstr;
+ s8 instr;
+};
+
+struct alt_group {
+ /*
+ * Pointer from a replacement group to the original group. NULL if it
+ * *is* the original group.
+ */
+ struct alt_group *orig_group;
+
+ /* First and last instructions in the group */
+ struct instruction *first_insn, *last_insn;
+
+ /*
+ * Byte-offset-addressed len-sized array of pointers to CFI structs.
+ * This is shared with the other alt_groups in the same alternative.
+ */
+ struct cfi_state **cfi;
+};
+
+struct instruction {
+ struct list_head list;
+ struct hlist_node hash;
+ struct list_head call_node;
+ struct section *sec;
+ unsigned long offset;
+ unsigned int len;
+ enum insn_type type;
+ unsigned long immediate;
+
+ u16 dead_end : 1,
+ ignore : 1,
+ ignore_alts : 1,
+ hint : 1,
+ save : 1,
+ restore : 1,
+ retpoline_safe : 1,
+ noendbr : 1,
+ entry : 1;
+ /* 7 bit hole */
+
+ s8 instr;
+ u8 visited;
+
+ struct alt_group *alt_group;
+ struct symbol *call_dest;
+ struct instruction *jump_dest;
+ struct instruction *first_jump_src;
+ struct reloc *jump_table;
+ struct reloc *reloc;
+ struct list_head alts;
+ struct symbol *func;
+ struct list_head stack_ops;
+ struct cfi_state *cfi;
+};
+
+#define VISITED_BRANCH 0x01
+#define VISITED_BRANCH_UACCESS 0x02
+#define VISITED_BRANCH_MASK 0x03
+#define VISITED_ENTRY 0x04
+
+static inline bool is_static_jump(struct instruction *insn)
+{
+ return insn->type == INSN_JUMP_CONDITIONAL ||
+ insn->type == INSN_JUMP_UNCONDITIONAL;
+}
+
+static inline bool is_dynamic_jump(struct instruction *insn)
+{
+ return insn->type == INSN_JUMP_DYNAMIC ||
+ insn->type == INSN_JUMP_DYNAMIC_CONDITIONAL;
+}
+
+static inline bool is_jump(struct instruction *insn)
+{
+ return is_static_jump(insn) || is_dynamic_jump(insn);
+}
+
+struct instruction *find_insn(struct objtool_file *file,
+ struct section *sec, unsigned long offset);
+
+#define for_each_insn(file, insn) \
+ list_for_each_entry(insn, &file->insn_list, list)
+
+#define sec_for_each_insn(file, sec, insn) \
+ for (insn = find_insn(file, sec, 0); \
+ insn && &insn->list != &file->insn_list && \
+ insn->sec == sec; \
+ insn = list_next_entry(insn, list))
+
+#endif /* _CHECK_H */
diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h
new file mode 100644
index 000000000..5d4a841fb
--- /dev/null
+++ b/tools/objtool/include/objtool/elf.h
@@ -0,0 +1,176 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com>
+ */
+
+#ifndef _OBJTOOL_ELF_H
+#define _OBJTOOL_ELF_H
+
+#include <stdio.h>
+#include <gelf.h>
+#include <linux/list.h>
+#include <linux/hashtable.h>
+#include <linux/rbtree.h>
+#include <linux/jhash.h>
+
+#ifdef LIBELF_USE_DEPRECATED
+# define elf_getshdrnum elf_getshnum
+# define elf_getshdrstrndx elf_getshstrndx
+#endif
+
+/*
+ * Fallback for systems without this "read, mmaping if possible" cmd.
+ */
+#ifndef ELF_C_READ_MMAP
+#define ELF_C_READ_MMAP ELF_C_READ
+#endif
+
+struct section {
+ struct list_head list;
+ struct hlist_node hash;
+ struct hlist_node name_hash;
+ GElf_Shdr sh;
+ struct rb_root symbol_tree;
+ struct list_head symbol_list;
+ struct list_head reloc_list;
+ struct section *base, *reloc;
+ struct symbol *sym;
+ Elf_Data *data;
+ char *name;
+ int idx;
+ bool changed, text, rodata, noinstr;
+};
+
+struct symbol {
+ struct list_head list;
+ struct rb_node node;
+ struct hlist_node hash;
+ struct hlist_node name_hash;
+ GElf_Sym sym;
+ struct section *sec;
+ char *name;
+ unsigned int idx;
+ unsigned char bind, type;
+ unsigned long offset;
+ unsigned int len;
+ struct symbol *pfunc, *cfunc, *alias;
+ u8 uaccess_safe : 1;
+ u8 static_call_tramp : 1;
+ u8 retpoline_thunk : 1;
+ u8 return_thunk : 1;
+ u8 fentry : 1;
+ u8 profiling_func : 1;
+ u8 embedded_insn : 1;
+ struct list_head pv_target;
+};
+
+struct reloc {
+ struct list_head list;
+ struct hlist_node hash;
+ union {
+ GElf_Rela rela;
+ GElf_Rel rel;
+ };
+ struct section *sec;
+ struct symbol *sym;
+ unsigned long offset;
+ unsigned int type;
+ s64 addend;
+ int idx;
+ bool jump_table_start;
+};
+
+#define ELF_HASH_BITS 20
+
+struct elf {
+ Elf *elf;
+ GElf_Ehdr ehdr;
+ int fd;
+ bool changed;
+ char *name;
+ unsigned int text_size, num_files;
+ struct list_head sections;
+
+ int symbol_bits;
+ int symbol_name_bits;
+ int section_bits;
+ int section_name_bits;
+ int reloc_bits;
+
+ struct hlist_head *symbol_hash;
+ struct hlist_head *symbol_name_hash;
+ struct hlist_head *section_hash;
+ struct hlist_head *section_name_hash;
+ struct hlist_head *reloc_hash;
+};
+
+#define OFFSET_STRIDE_BITS 4
+#define OFFSET_STRIDE (1UL << OFFSET_STRIDE_BITS)
+#define OFFSET_STRIDE_MASK (~(OFFSET_STRIDE - 1))
+
+#define for_offset_range(_offset, _start, _end) \
+ for (_offset = ((_start) & OFFSET_STRIDE_MASK); \
+ _offset >= ((_start) & OFFSET_STRIDE_MASK) && \
+ _offset <= ((_end) & OFFSET_STRIDE_MASK); \
+ _offset += OFFSET_STRIDE)
+
+static inline u32 sec_offset_hash(struct section *sec, unsigned long offset)
+{
+ u32 ol, oh, idx = sec->idx;
+
+ offset &= OFFSET_STRIDE_MASK;
+
+ ol = offset;
+ oh = (offset >> 16) >> 16;
+
+ __jhash_mix(ol, oh, idx);
+
+ return ol;
+}
+
+static inline u32 reloc_hash(struct reloc *reloc)
+{
+ return sec_offset_hash(reloc->sec, reloc->offset);
+}
+
+/*
+ * Try to see if it's a whole archive (vmlinux.o or module).
+ *
+ * Note this will miss the case where a module only has one source file.
+ */
+static inline bool has_multiple_files(struct elf *elf)
+{
+ return elf->num_files > 1;
+}
+
+struct elf *elf_open_read(const char *name, int flags);
+struct section *elf_create_section(struct elf *elf, const char *name, unsigned int sh_flags, size_t entsize, int nr);
+
+int elf_add_reloc(struct elf *elf, struct section *sec, unsigned long offset,
+ unsigned int type, struct symbol *sym, s64 addend);
+int elf_add_reloc_to_insn(struct elf *elf, struct section *sec,
+ unsigned long offset, unsigned int type,
+ struct section *insn_sec, unsigned long insn_off);
+
+int elf_write_insn(struct elf *elf, struct section *sec,
+ unsigned long offset, unsigned int len,
+ const char *insn);
+int elf_write_reloc(struct elf *elf, struct reloc *reloc);
+int elf_write(struct elf *elf);
+void elf_close(struct elf *elf);
+
+struct section *find_section_by_name(const struct elf *elf, const char *name);
+struct symbol *find_func_by_offset(struct section *sec, unsigned long offset);
+struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset);
+struct symbol *find_symbol_by_name(const struct elf *elf, const char *name);
+struct symbol *find_symbol_containing(const struct section *sec, unsigned long offset);
+int find_symbol_hole_containing(const struct section *sec, unsigned long offset);
+struct reloc *find_reloc_by_dest(const struct elf *elf, struct section *sec, unsigned long offset);
+struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *sec,
+ unsigned long offset, unsigned int len);
+struct symbol *find_func_containing(struct section *sec, unsigned long offset);
+
+#define for_each_sec(file, sec) \
+ list_for_each_entry(sec, &file->elf->sections, list)
+
+#endif /* _OBJTOOL_ELF_H */
diff --git a/tools/objtool/include/objtool/endianness.h b/tools/objtool/include/objtool/endianness.h
new file mode 100644
index 000000000..10241341e
--- /dev/null
+++ b/tools/objtool/include/objtool/endianness.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _OBJTOOL_ENDIANNESS_H
+#define _OBJTOOL_ENDIANNESS_H
+
+#include <arch/endianness.h>
+#include <linux/kernel.h>
+#include <endian.h>
+
+#ifndef __TARGET_BYTE_ORDER
+#error undefined arch __TARGET_BYTE_ORDER
+#endif
+
+#if __BYTE_ORDER != __TARGET_BYTE_ORDER
+#define __NEED_BSWAP 1
+#else
+#define __NEED_BSWAP 0
+#endif
+
+/*
+ * Does a byte swap if target endianness doesn't match the host, i.e. cross
+ * compilation for little endian on big endian and vice versa.
+ * To be used for multi-byte values conversion, which are read from / about
+ * to be written to a target native endianness ELF file.
+ */
+#define bswap_if_needed(val) \
+({ \
+ __typeof__(val) __ret; \
+ switch (sizeof(val)) { \
+ case 8: __ret = __NEED_BSWAP ? bswap_64(val) : (val); break; \
+ case 4: __ret = __NEED_BSWAP ? bswap_32(val) : (val); break; \
+ case 2: __ret = __NEED_BSWAP ? bswap_16(val) : (val); break; \
+ default: \
+ BUILD_BUG(); break; \
+ } \
+ __ret; \
+})
+
+#endif /* _OBJTOOL_ENDIANNESS_H */
diff --git a/tools/objtool/include/objtool/objtool.h b/tools/objtool/include/objtool/objtool.h
new file mode 100644
index 000000000..7f2d1b095
--- /dev/null
+++ b/tools/objtool/include/objtool/objtool.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2020 Matt Helsley <mhelsley@vmware.com>
+ */
+
+#ifndef _OBJTOOL_H
+#define _OBJTOOL_H
+
+#include <stdbool.h>
+#include <linux/list.h>
+#include <linux/hashtable.h>
+
+#include <objtool/elf.h>
+
+#define __weak __attribute__((weak))
+
+struct pv_state {
+ bool clean;
+ struct list_head targets;
+};
+
+struct objtool_file {
+ struct elf *elf;
+ struct list_head insn_list;
+ DECLARE_HASHTABLE(insn_hash, 20);
+ struct list_head retpoline_call_list;
+ struct list_head return_thunk_list;
+ struct list_head static_call_list;
+ struct list_head mcount_loc_list;
+ struct list_head endbr_list;
+ bool ignore_unreachables, hints, rodata;
+
+ unsigned int nr_endbr;
+ unsigned int nr_endbr_int;
+
+ unsigned long jl_short, jl_long;
+ unsigned long jl_nop_short, jl_nop_long;
+
+ struct pv_state *pv_ops;
+};
+
+struct objtool_file *objtool_open_read(const char *_objname);
+
+void objtool_pv_add(struct objtool_file *file, int idx, struct symbol *func);
+
+int check(struct objtool_file *file);
+int orc_dump(const char *objname);
+int orc_create(struct objtool_file *file);
+
+#endif /* _OBJTOOL_H */
diff --git a/tools/objtool/include/objtool/special.h b/tools/objtool/include/objtool/special.h
new file mode 100644
index 000000000..dc4721e19
--- /dev/null
+++ b/tools/objtool/include/objtool/special.h
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com>
+ */
+
+#ifndef _SPECIAL_H
+#define _SPECIAL_H
+
+#include <stdbool.h>
+#include <objtool/check.h>
+#include <objtool/elf.h>
+
+#define C_JUMP_TABLE_SECTION ".rodata..c_jump_table"
+
+struct special_alt {
+ struct list_head list;
+
+ bool group;
+ bool skip_orig;
+ bool skip_alt;
+ bool jump_or_nop;
+
+ struct section *orig_sec;
+ unsigned long orig_off;
+
+ struct section *new_sec;
+ unsigned long new_off;
+
+ unsigned int orig_len, new_len; /* group only */
+ u8 key_addend;
+};
+
+int special_get_alts(struct elf *elf, struct list_head *alts);
+
+void arch_handle_alternative(unsigned short feature, struct special_alt *alt);
+
+bool arch_support_alt_relocation(struct special_alt *special_alt,
+ struct instruction *insn,
+ struct reloc *reloc);
+struct reloc *arch_find_switch_table(struct objtool_file *file,
+ struct instruction *insn);
+#endif /* _SPECIAL_H */
diff --git a/tools/objtool/include/objtool/warn.h b/tools/objtool/include/objtool/warn.h
new file mode 100644
index 000000000..a3e79ae75
--- /dev/null
+++ b/tools/objtool/include/objtool/warn.h
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com>
+ */
+
+#ifndef _WARN_H
+#define _WARN_H
+
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <objtool/builtin.h>
+#include <objtool/elf.h>
+
+extern const char *objname;
+
+static inline char *offstr(struct section *sec, unsigned long offset)
+{
+ bool is_text = (sec->sh.sh_flags & SHF_EXECINSTR);
+ struct symbol *sym = NULL;
+ char *str;
+ int len;
+
+ if (is_text)
+ sym = find_func_containing(sec, offset);
+ if (!sym)
+ sym = find_symbol_containing(sec, offset);
+
+ if (sym) {
+ str = malloc(strlen(sym->name) + strlen(sec->name) + 40);
+ len = sprintf(str, "%s+0x%lx", sym->name, offset - sym->offset);
+ if (opts.sec_address)
+ sprintf(str+len, " (%s+0x%lx)", sec->name, offset);
+ } else {
+ str = malloc(strlen(sec->name) + 20);
+ sprintf(str, "%s+0x%lx", sec->name, offset);
+ }
+
+ return str;
+}
+
+#define WARN(format, ...) \
+ fprintf(stderr, \
+ "%s: warning: objtool: " format "\n", \
+ objname, ##__VA_ARGS__)
+
+#define WARN_FUNC(format, sec, offset, ...) \
+({ \
+ char *_str = offstr(sec, offset); \
+ WARN("%s: " format, _str, ##__VA_ARGS__); \
+ free(_str); \
+})
+
+#define BT_FUNC(format, insn, ...) \
+({ \
+ struct instruction *_insn = (insn); \
+ char *_str = offstr(_insn->sec, _insn->offset); \
+ WARN(" %s: " format, _str, ##__VA_ARGS__); \
+ free(_str); \
+})
+
+#define WARN_ELF(format, ...) \
+ WARN(format ": %s", ##__VA_ARGS__, elf_errmsg(-1))
+
+#endif /* _WARN_H */
diff --git a/tools/objtool/objtool.c b/tools/objtool/objtool.c
new file mode 100644
index 000000000..cda649644
--- /dev/null
+++ b/tools/objtool/objtool.c
@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com>
+ */
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <subcmd/exec-cmd.h>
+#include <subcmd/pager.h>
+#include <linux/kernel.h>
+
+#include <objtool/builtin.h>
+#include <objtool/objtool.h>
+#include <objtool/warn.h>
+
+bool help;
+
+const char *objname;
+static struct objtool_file file;
+
+static bool objtool_create_backup(const char *_objname)
+{
+ int len = strlen(_objname);
+ char *buf, *base, *name = malloc(len+6);
+ int s, d, l, t;
+
+ if (!name) {
+ perror("failed backup name malloc");
+ return false;
+ }
+
+ strcpy(name, _objname);
+ strcpy(name + len, ".orig");
+
+ d = open(name, O_CREAT|O_WRONLY|O_TRUNC, 0644);
+ if (d < 0) {
+ perror("failed to create backup file");
+ return false;
+ }
+
+ s = open(_objname, O_RDONLY);
+ if (s < 0) {
+ perror("failed to open orig file");
+ return false;
+ }
+
+ buf = malloc(4096);
+ if (!buf) {
+ perror("failed backup data malloc");
+ return false;
+ }
+
+ while ((l = read(s, buf, 4096)) > 0) {
+ base = buf;
+ do {
+ t = write(d, base, l);
+ if (t < 0) {
+ perror("failed backup write");
+ return false;
+ }
+ base += t;
+ l -= t;
+ } while (l);
+ }
+
+ if (l < 0) {
+ perror("failed backup read");
+ return false;
+ }
+
+ free(name);
+ free(buf);
+ close(d);
+ close(s);
+
+ return true;
+}
+
+struct objtool_file *objtool_open_read(const char *_objname)
+{
+ if (objname) {
+ if (strcmp(objname, _objname)) {
+ WARN("won't handle more than one file at a time");
+ return NULL;
+ }
+ return &file;
+ }
+ objname = _objname;
+
+ file.elf = elf_open_read(objname, O_RDWR);
+ if (!file.elf)
+ return NULL;
+
+ if (opts.backup && !objtool_create_backup(objname)) {
+ WARN("can't create backup file");
+ return NULL;
+ }
+
+ INIT_LIST_HEAD(&file.insn_list);
+ hash_init(file.insn_hash);
+ INIT_LIST_HEAD(&file.retpoline_call_list);
+ INIT_LIST_HEAD(&file.return_thunk_list);
+ INIT_LIST_HEAD(&file.static_call_list);
+ INIT_LIST_HEAD(&file.mcount_loc_list);
+ INIT_LIST_HEAD(&file.endbr_list);
+ file.ignore_unreachables = opts.no_unreachable;
+ file.hints = false;
+
+ return &file;
+}
+
+void objtool_pv_add(struct objtool_file *f, int idx, struct symbol *func)
+{
+ if (!opts.noinstr)
+ return;
+
+ if (!f->pv_ops) {
+ WARN("paravirt confusion");
+ return;
+ }
+
+ /*
+ * These functions will be patched into native code,
+ * see paravirt_patch().
+ */
+ if (!strcmp(func->name, "_paravirt_nop") ||
+ !strcmp(func->name, "_paravirt_ident_64"))
+ return;
+
+ /* already added this function */
+ if (!list_empty(&func->pv_target))
+ return;
+
+ list_add(&func->pv_target, &f->pv_ops[idx].targets);
+ f->pv_ops[idx].clean = false;
+}
+
+int main(int argc, const char **argv)
+{
+ static const char *UNUSED = "OBJTOOL_NOT_IMPLEMENTED";
+
+ /* libsubcmd init */
+ exec_cmd_init("objtool", UNUSED, UNUSED, UNUSED);
+ pager_init(UNUSED);
+
+ return objtool_run(argc, argv);
+}
diff --git a/tools/objtool/orc_dump.c b/tools/objtool/orc_dump.c
new file mode 100644
index 000000000..f5a8508c4
--- /dev/null
+++ b/tools/objtool/orc_dump.c
@@ -0,0 +1,215 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
+ */
+
+#include <unistd.h>
+#include <linux/objtool.h>
+#include <asm/orc_types.h>
+#include <objtool/objtool.h>
+#include <objtool/warn.h>
+#include <objtool/endianness.h>
+
+static const char *reg_name(unsigned int reg)
+{
+ switch (reg) {
+ case ORC_REG_PREV_SP:
+ return "prevsp";
+ case ORC_REG_DX:
+ return "dx";
+ case ORC_REG_DI:
+ return "di";
+ case ORC_REG_BP:
+ return "bp";
+ case ORC_REG_SP:
+ return "sp";
+ case ORC_REG_R10:
+ return "r10";
+ case ORC_REG_R13:
+ return "r13";
+ case ORC_REG_BP_INDIRECT:
+ return "bp(ind)";
+ case ORC_REG_SP_INDIRECT:
+ return "sp(ind)";
+ default:
+ return "?";
+ }
+}
+
+static const char *orc_type_name(unsigned int type)
+{
+ switch (type) {
+ case UNWIND_HINT_TYPE_CALL:
+ return "call";
+ case UNWIND_HINT_TYPE_REGS:
+ return "regs";
+ case UNWIND_HINT_TYPE_REGS_PARTIAL:
+ return "regs (partial)";
+ default:
+ return "?";
+ }
+}
+
+static void print_reg(unsigned int reg, int offset)
+{
+ if (reg == ORC_REG_BP_INDIRECT)
+ printf("(bp%+d)", offset);
+ else if (reg == ORC_REG_SP_INDIRECT)
+ printf("(sp)%+d", offset);
+ else if (reg == ORC_REG_UNDEFINED)
+ printf("(und)");
+ else
+ printf("%s%+d", reg_name(reg), offset);
+}
+
+int orc_dump(const char *_objname)
+{
+ int fd, nr_entries, i, *orc_ip = NULL, orc_size = 0;
+ struct orc_entry *orc = NULL;
+ char *name;
+ size_t nr_sections;
+ Elf64_Addr orc_ip_addr = 0;
+ size_t shstrtab_idx, strtab_idx = 0;
+ Elf *elf;
+ Elf_Scn *scn;
+ GElf_Shdr sh;
+ GElf_Rela rela;
+ GElf_Sym sym;
+ Elf_Data *data, *symtab = NULL, *rela_orc_ip = NULL;
+
+
+ objname = _objname;
+
+ elf_version(EV_CURRENT);
+
+ fd = open(objname, O_RDONLY);
+ if (fd == -1) {
+ perror("open");
+ return -1;
+ }
+
+ elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);
+ if (!elf) {
+ WARN_ELF("elf_begin");
+ return -1;
+ }
+
+ if (elf_getshdrnum(elf, &nr_sections)) {
+ WARN_ELF("elf_getshdrnum");
+ return -1;
+ }
+
+ if (elf_getshdrstrndx(elf, &shstrtab_idx)) {
+ WARN_ELF("elf_getshdrstrndx");
+ return -1;
+ }
+
+ for (i = 0; i < nr_sections; i++) {
+ scn = elf_getscn(elf, i);
+ if (!scn) {
+ WARN_ELF("elf_getscn");
+ return -1;
+ }
+
+ if (!gelf_getshdr(scn, &sh)) {
+ WARN_ELF("gelf_getshdr");
+ return -1;
+ }
+
+ name = elf_strptr(elf, shstrtab_idx, sh.sh_name);
+ if (!name) {
+ WARN_ELF("elf_strptr");
+ return -1;
+ }
+
+ data = elf_getdata(scn, NULL);
+ if (!data) {
+ WARN_ELF("elf_getdata");
+ return -1;
+ }
+
+ if (!strcmp(name, ".symtab")) {
+ symtab = data;
+ } else if (!strcmp(name, ".strtab")) {
+ strtab_idx = i;
+ } else if (!strcmp(name, ".orc_unwind")) {
+ orc = data->d_buf;
+ orc_size = sh.sh_size;
+ } else if (!strcmp(name, ".orc_unwind_ip")) {
+ orc_ip = data->d_buf;
+ orc_ip_addr = sh.sh_addr;
+ } else if (!strcmp(name, ".rela.orc_unwind_ip")) {
+ rela_orc_ip = data;
+ }
+ }
+
+ if (!symtab || !strtab_idx || !orc || !orc_ip)
+ return 0;
+
+ if (orc_size % sizeof(*orc) != 0) {
+ WARN("bad .orc_unwind section size");
+ return -1;
+ }
+
+ nr_entries = orc_size / sizeof(*orc);
+ for (i = 0; i < nr_entries; i++) {
+ if (rela_orc_ip) {
+ if (!gelf_getrela(rela_orc_ip, i, &rela)) {
+ WARN_ELF("gelf_getrela");
+ return -1;
+ }
+
+ if (!gelf_getsym(symtab, GELF_R_SYM(rela.r_info), &sym)) {
+ WARN_ELF("gelf_getsym");
+ return -1;
+ }
+
+ if (GELF_ST_TYPE(sym.st_info) == STT_SECTION) {
+ scn = elf_getscn(elf, sym.st_shndx);
+ if (!scn) {
+ WARN_ELF("elf_getscn");
+ return -1;
+ }
+
+ if (!gelf_getshdr(scn, &sh)) {
+ WARN_ELF("gelf_getshdr");
+ return -1;
+ }
+
+ name = elf_strptr(elf, shstrtab_idx, sh.sh_name);
+ if (!name) {
+ WARN_ELF("elf_strptr");
+ return -1;
+ }
+ } else {
+ name = elf_strptr(elf, strtab_idx, sym.st_name);
+ if (!name) {
+ WARN_ELF("elf_strptr");
+ return -1;
+ }
+ }
+
+ printf("%s+%llx:", name, (unsigned long long)rela.r_addend);
+
+ } else {
+ printf("%llx:", (unsigned long long)(orc_ip_addr + (i * sizeof(int)) + orc_ip[i]));
+ }
+
+
+ printf(" sp:");
+
+ print_reg(orc[i].sp_reg, bswap_if_needed(orc[i].sp_offset));
+
+ printf(" bp:");
+
+ print_reg(orc[i].bp_reg, bswap_if_needed(orc[i].bp_offset));
+
+ printf(" type:%s end:%d\n",
+ orc_type_name(orc[i].type), orc[i].end);
+ }
+
+ elf_end(elf);
+ close(fd);
+
+ return 0;
+}
diff --git a/tools/objtool/orc_gen.c b/tools/objtool/orc_gen.c
new file mode 100644
index 000000000..dd3c64af9
--- /dev/null
+++ b/tools/objtool/orc_gen.c
@@ -0,0 +1,245 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <linux/objtool.h>
+#include <asm/orc_types.h>
+
+#include <objtool/check.h>
+#include <objtool/warn.h>
+#include <objtool/endianness.h>
+
+static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi,
+ struct instruction *insn)
+{
+ struct cfi_reg *bp = &cfi->regs[CFI_BP];
+
+ memset(orc, 0, sizeof(*orc));
+
+ if (!cfi) {
+ orc->end = 0;
+ orc->sp_reg = ORC_REG_UNDEFINED;
+ return 0;
+ }
+
+ orc->end = cfi->end;
+
+ if (cfi->cfa.base == CFI_UNDEFINED) {
+ orc->sp_reg = ORC_REG_UNDEFINED;
+ return 0;
+ }
+
+ switch (cfi->cfa.base) {
+ case CFI_SP:
+ orc->sp_reg = ORC_REG_SP;
+ break;
+ case CFI_SP_INDIRECT:
+ orc->sp_reg = ORC_REG_SP_INDIRECT;
+ break;
+ case CFI_BP:
+ orc->sp_reg = ORC_REG_BP;
+ break;
+ case CFI_BP_INDIRECT:
+ orc->sp_reg = ORC_REG_BP_INDIRECT;
+ break;
+ case CFI_R10:
+ orc->sp_reg = ORC_REG_R10;
+ break;
+ case CFI_R13:
+ orc->sp_reg = ORC_REG_R13;
+ break;
+ case CFI_DI:
+ orc->sp_reg = ORC_REG_DI;
+ break;
+ case CFI_DX:
+ orc->sp_reg = ORC_REG_DX;
+ break;
+ default:
+ WARN_FUNC("unknown CFA base reg %d",
+ insn->sec, insn->offset, cfi->cfa.base);
+ return -1;
+ }
+
+ switch (bp->base) {
+ case CFI_UNDEFINED:
+ orc->bp_reg = ORC_REG_UNDEFINED;
+ break;
+ case CFI_CFA:
+ orc->bp_reg = ORC_REG_PREV_SP;
+ break;
+ case CFI_BP:
+ orc->bp_reg = ORC_REG_BP;
+ break;
+ default:
+ WARN_FUNC("unknown BP base reg %d",
+ insn->sec, insn->offset, bp->base);
+ return -1;
+ }
+
+ orc->sp_offset = cfi->cfa.offset;
+ orc->bp_offset = bp->offset;
+ orc->type = cfi->type;
+
+ return 0;
+}
+
+static int write_orc_entry(struct elf *elf, struct section *orc_sec,
+ struct section *ip_sec, unsigned int idx,
+ struct section *insn_sec, unsigned long insn_off,
+ struct orc_entry *o)
+{
+ struct orc_entry *orc;
+
+ /* populate ORC data */
+ orc = (struct orc_entry *)orc_sec->data->d_buf + idx;
+ memcpy(orc, o, sizeof(*orc));
+ orc->sp_offset = bswap_if_needed(orc->sp_offset);
+ orc->bp_offset = bswap_if_needed(orc->bp_offset);
+
+ /* populate reloc for ip */
+ if (elf_add_reloc_to_insn(elf, ip_sec, idx * sizeof(int), R_X86_64_PC32,
+ insn_sec, insn_off))
+ return -1;
+
+ return 0;
+}
+
+struct orc_list_entry {
+ struct list_head list;
+ struct orc_entry orc;
+ struct section *insn_sec;
+ unsigned long insn_off;
+};
+
+static int orc_list_add(struct list_head *orc_list, struct orc_entry *orc,
+ struct section *sec, unsigned long offset)
+{
+ struct orc_list_entry *entry = malloc(sizeof(*entry));
+
+ if (!entry) {
+ WARN("malloc failed");
+ return -1;
+ }
+
+ entry->orc = *orc;
+ entry->insn_sec = sec;
+ entry->insn_off = offset;
+
+ list_add_tail(&entry->list, orc_list);
+ return 0;
+}
+
+static unsigned long alt_group_len(struct alt_group *alt_group)
+{
+ return alt_group->last_insn->offset +
+ alt_group->last_insn->len -
+ alt_group->first_insn->offset;
+}
+
+int orc_create(struct objtool_file *file)
+{
+ struct section *sec, *orc_sec;
+ unsigned int nr = 0, idx = 0;
+ struct orc_list_entry *entry;
+ struct list_head orc_list;
+
+ struct orc_entry null = {
+ .sp_reg = ORC_REG_UNDEFINED,
+ .bp_reg = ORC_REG_UNDEFINED,
+ .type = UNWIND_HINT_TYPE_CALL,
+ };
+
+ /* Build a deduplicated list of ORC entries: */
+ INIT_LIST_HEAD(&orc_list);
+ for_each_sec(file, sec) {
+ struct orc_entry orc, prev_orc = {0};
+ struct instruction *insn;
+ bool empty = true;
+
+ if (!sec->text)
+ continue;
+
+ sec_for_each_insn(file, sec, insn) {
+ struct alt_group *alt_group = insn->alt_group;
+ int i;
+
+ if (!alt_group) {
+ if (init_orc_entry(&orc, insn->cfi, insn))
+ return -1;
+ if (!memcmp(&prev_orc, &orc, sizeof(orc)))
+ continue;
+ if (orc_list_add(&orc_list, &orc, sec,
+ insn->offset))
+ return -1;
+ nr++;
+ prev_orc = orc;
+ empty = false;
+ continue;
+ }
+
+ /*
+ * Alternatives can have different stack layout
+ * possibilities (but they shouldn't conflict).
+ * Instead of traversing the instructions, use the
+ * alt_group's flattened byte-offset-addressed CFI
+ * array.
+ */
+ for (i = 0; i < alt_group_len(alt_group); i++) {
+ struct cfi_state *cfi = alt_group->cfi[i];
+ if (!cfi)
+ continue;
+ /* errors are reported on the original insn */
+ if (init_orc_entry(&orc, cfi, insn))
+ return -1;
+ if (!memcmp(&prev_orc, &orc, sizeof(orc)))
+ continue;
+ if (orc_list_add(&orc_list, &orc, insn->sec,
+ insn->offset + i))
+ return -1;
+ nr++;
+ prev_orc = orc;
+ empty = false;
+ }
+
+ /* Skip to the end of the alt_group */
+ insn = alt_group->last_insn;
+ }
+
+ /* Add a section terminator */
+ if (!empty) {
+ orc_list_add(&orc_list, &null, sec, sec->sh.sh_size);
+ nr++;
+ }
+ }
+ if (!nr)
+ return 0;
+
+ /* Create .orc_unwind, .orc_unwind_ip and .rela.orc_unwind_ip sections: */
+ sec = find_section_by_name(file->elf, ".orc_unwind");
+ if (sec) {
+ WARN("file already has .orc_unwind section, skipping");
+ return -1;
+ }
+ orc_sec = elf_create_section(file->elf, ".orc_unwind", 0,
+ sizeof(struct orc_entry), nr);
+ if (!orc_sec)
+ return -1;
+
+ sec = elf_create_section(file->elf, ".orc_unwind_ip", 0, sizeof(int), nr);
+ if (!sec)
+ return -1;
+
+ /* Write ORC entries to sections: */
+ list_for_each_entry(entry, &orc_list, list) {
+ if (write_orc_entry(file->elf, orc_sec, sec, idx++,
+ entry->insn_sec, entry->insn_off,
+ &entry->orc))
+ return -1;
+ }
+
+ return 0;
+}
diff --git a/tools/objtool/special.c b/tools/objtool/special.c
new file mode 100644
index 000000000..e2223dd91
--- /dev/null
+++ b/tools/objtool/special.c
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com>
+ */
+
+/*
+ * This file reads all the special sections which have alternate instructions
+ * which can be patched in or redirected to at runtime.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <arch/special.h>
+#include <objtool/builtin.h>
+#include <objtool/special.h>
+#include <objtool/warn.h>
+#include <objtool/endianness.h>
+
+struct special_entry {
+ const char *sec;
+ bool group, jump_or_nop;
+ unsigned char size, orig, new;
+ unsigned char orig_len, new_len; /* group only */
+ unsigned char feature; /* ALTERNATIVE macro CPU feature */
+ unsigned char key; /* jump_label key */
+};
+
+struct special_entry entries[] = {
+ {
+ .sec = ".altinstructions",
+ .group = true,
+ .size = ALT_ENTRY_SIZE,
+ .orig = ALT_ORIG_OFFSET,
+ .orig_len = ALT_ORIG_LEN_OFFSET,
+ .new = ALT_NEW_OFFSET,
+ .new_len = ALT_NEW_LEN_OFFSET,
+ .feature = ALT_FEATURE_OFFSET,
+ },
+ {
+ .sec = "__jump_table",
+ .jump_or_nop = true,
+ .size = JUMP_ENTRY_SIZE,
+ .orig = JUMP_ORIG_OFFSET,
+ .new = JUMP_NEW_OFFSET,
+ .key = JUMP_KEY_OFFSET,
+ },
+ {
+ .sec = "__ex_table",
+ .size = EX_ENTRY_SIZE,
+ .orig = EX_ORIG_OFFSET,
+ .new = EX_NEW_OFFSET,
+ },
+ {},
+};
+
+void __weak arch_handle_alternative(unsigned short feature, struct special_alt *alt)
+{
+}
+
+static void reloc_to_sec_off(struct reloc *reloc, struct section **sec,
+ unsigned long *off)
+{
+ *sec = reloc->sym->sec;
+ *off = reloc->sym->offset + reloc->addend;
+}
+
+static int get_alt_entry(struct elf *elf, struct special_entry *entry,
+ struct section *sec, int idx,
+ struct special_alt *alt)
+{
+ struct reloc *orig_reloc, *new_reloc;
+ unsigned long offset;
+
+ offset = idx * entry->size;
+
+ alt->group = entry->group;
+ alt->jump_or_nop = entry->jump_or_nop;
+
+ if (alt->group) {
+ alt->orig_len = *(unsigned char *)(sec->data->d_buf + offset +
+ entry->orig_len);
+ alt->new_len = *(unsigned char *)(sec->data->d_buf + offset +
+ entry->new_len);
+ }
+
+ if (entry->feature) {
+ unsigned short feature;
+
+ feature = bswap_if_needed(*(unsigned short *)(sec->data->d_buf +
+ offset +
+ entry->feature));
+ arch_handle_alternative(feature, alt);
+ }
+
+ orig_reloc = find_reloc_by_dest(elf, sec, offset + entry->orig);
+ if (!orig_reloc) {
+ WARN_FUNC("can't find orig reloc", sec, offset + entry->orig);
+ return -1;
+ }
+
+ reloc_to_sec_off(orig_reloc, &alt->orig_sec, &alt->orig_off);
+
+ if (!entry->group || alt->new_len) {
+ new_reloc = find_reloc_by_dest(elf, sec, offset + entry->new);
+ if (!new_reloc) {
+ WARN_FUNC("can't find new reloc",
+ sec, offset + entry->new);
+ return -1;
+ }
+
+ reloc_to_sec_off(new_reloc, &alt->new_sec, &alt->new_off);
+
+ /* _ASM_EXTABLE_EX hack */
+ if (alt->new_off >= 0x7ffffff0)
+ alt->new_off -= 0x7ffffff0;
+ }
+
+ if (entry->key) {
+ struct reloc *key_reloc;
+
+ key_reloc = find_reloc_by_dest(elf, sec, offset + entry->key);
+ if (!key_reloc) {
+ WARN_FUNC("can't find key reloc",
+ sec, offset + entry->key);
+ return -1;
+ }
+ alt->key_addend = key_reloc->addend;
+ }
+
+ return 0;
+}
+
+/*
+ * Read all the special sections and create a list of special_alt structs which
+ * describe all the alternate instructions which can be patched in or
+ * redirected to at runtime.
+ */
+int special_get_alts(struct elf *elf, struct list_head *alts)
+{
+ struct special_entry *entry;
+ struct section *sec;
+ unsigned int nr_entries;
+ struct special_alt *alt;
+ int idx, ret;
+
+ INIT_LIST_HEAD(alts);
+
+ for (entry = entries; entry->sec; entry++) {
+ sec = find_section_by_name(elf, entry->sec);
+ if (!sec)
+ continue;
+
+ if (sec->sh.sh_size % entry->size != 0) {
+ WARN("%s size not a multiple of %d",
+ sec->name, entry->size);
+ return -1;
+ }
+
+ nr_entries = sec->sh.sh_size / entry->size;
+
+ for (idx = 0; idx < nr_entries; idx++) {
+ alt = malloc(sizeof(*alt));
+ if (!alt) {
+ WARN("malloc failed");
+ return -1;
+ }
+ memset(alt, 0, sizeof(*alt));
+
+ ret = get_alt_entry(elf, entry, sec, idx, alt);
+ if (ret > 0)
+ continue;
+ if (ret < 0)
+ return ret;
+
+ list_add_tail(&alt->list, alts);
+ }
+ }
+
+ return 0;
+}
diff --git a/tools/objtool/sync-check.sh b/tools/objtool/sync-check.sh
new file mode 100755
index 000000000..105a291ff
--- /dev/null
+++ b/tools/objtool/sync-check.sh
@@ -0,0 +1,75 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+
+if [ -z "$SRCARCH" ]; then
+ echo 'sync-check.sh: error: missing $SRCARCH environment variable' >&2
+ exit 1
+fi
+
+FILES="include/linux/objtool.h"
+
+if [ "$SRCARCH" = "x86" ]; then
+FILES="$FILES
+arch/x86/include/asm/nops.h
+arch/x86/include/asm/inat_types.h
+arch/x86/include/asm/orc_types.h
+arch/x86/include/asm/emulate_prefix.h
+arch/x86/lib/x86-opcode-map.txt
+arch/x86/tools/gen-insn-attr-x86.awk
+include/linux/static_call_types.h
+"
+
+SYNC_CHECK_FILES='
+arch/x86/include/asm/inat.h
+arch/x86/include/asm/insn.h
+arch/x86/lib/inat.c
+arch/x86/lib/insn.c
+'
+fi
+
+check_2 () {
+ file1=$1
+ file2=$2
+
+ shift
+ shift
+
+ cmd="diff $* $file1 $file2 > /dev/null"
+
+ test -f $file2 && {
+ eval $cmd || {
+ echo "Warning: Kernel ABI header at '$file1' differs from latest version at '$file2'" >&2
+ echo diff -u $file1 $file2
+ }
+ }
+}
+
+check () {
+ file=$1
+
+ shift
+
+ check_2 tools/$file $file $*
+}
+
+if [ ! -d ../../kernel ] || [ ! -d ../../tools ] || [ ! -d ../objtool ]; then
+ exit 0
+fi
+
+cd ../..
+
+while read -r file_entry; do
+ if [ -z "$file_entry" ]; then
+ continue
+ fi
+
+ check $file_entry
+done <<EOF
+$FILES
+EOF
+
+if [ "$SRCARCH" = "x86" ]; then
+ for i in $SYNC_CHECK_FILES; do
+ check $i '-I "^.*\/\*.*__ignore_sync_check__.*\*\/.*$"'
+ done
+fi
diff --git a/tools/objtool/weak.c b/tools/objtool/weak.c
new file mode 100644
index 000000000..d83f60773
--- /dev/null
+++ b/tools/objtool/weak.c
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2020 Matt Helsley <mhelsley@vmware.com>
+ * Weak definitions necessary to compile objtool without
+ * some subcommands (e.g. check, orc).
+ */
+
+#include <stdbool.h>
+#include <errno.h>
+#include <objtool/objtool.h>
+
+#define UNSUPPORTED(name) \
+({ \
+ fprintf(stderr, "error: objtool: " name " not implemented\n"); \
+ return ENOSYS; \
+})
+
+int __weak orc_dump(const char *_objname)
+{
+ UNSUPPORTED("ORC");
+}
+
+int __weak orc_create(struct objtool_file *file)
+{
+ UNSUPPORTED("ORC");
+}