diff options
Diffstat (limited to 'arch/x86/kernel/relocate_kernel_32.S')
| -rw-r--r-- | arch/x86/kernel/relocate_kernel_32.S | 291 |
1 files changed, 291 insertions, 0 deletions
diff --git a/arch/x86/kernel/relocate_kernel_32.S b/arch/x86/kernel/relocate_kernel_32.S new file mode 100644 index 000000000..c7c4b1917 --- /dev/null +++ b/arch/x86/kernel/relocate_kernel_32.S @@ -0,0 +1,291 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * relocate_kernel.S - put the kernel image in place to boot + * Copyright (C) 2002-2004 Eric Biederman <ebiederm@xmission.com> + */ + +#include <linux/linkage.h> +#include <asm/page_types.h> +#include <asm/kexec.h> +#include <asm/nospec-branch.h> +#include <asm/processor-flags.h> + +/* + * Must be relocatable PIC code callable as a C function, in particular + * there must be a plain RET and not jump to return thunk. + */ + +#define PTR(x) (x << 2) + +/* + * control_page + KEXEC_CONTROL_CODE_MAX_SIZE + * ~ control_page + PAGE_SIZE are used as data storage and stack for + * jumping back + */ +#define DATA(offset) (KEXEC_CONTROL_CODE_MAX_SIZE+(offset)) + +/* Minimal CPU state */ +#define ESP DATA(0x0) +#define CR0 DATA(0x4) +#define CR3 DATA(0x8) +#define CR4 DATA(0xc) + +/* other data */ +#define CP_VA_CONTROL_PAGE DATA(0x10) +#define CP_PA_PGD DATA(0x14) +#define CP_PA_SWAP_PAGE DATA(0x18) +#define CP_PA_BACKUP_PAGES_MAP DATA(0x1c) + + .text +SYM_CODE_START_NOALIGN(relocate_kernel) + /* Save the CPU context, used for jumping back */ + + pushl %ebx + pushl %esi + pushl %edi + pushl %ebp + pushf + + movl 20+8(%esp), %ebp /* list of pages */ + movl PTR(VA_CONTROL_PAGE)(%ebp), %edi + movl %esp, ESP(%edi) + movl %cr0, %eax + movl %eax, CR0(%edi) + movl %cr3, %eax + movl %eax, CR3(%edi) + movl %cr4, %eax + movl %eax, CR4(%edi) + + /* read the arguments and say goodbye to the stack */ + movl 20+4(%esp), %ebx /* page_list */ + movl 20+8(%esp), %ebp /* list of pages */ + movl 20+12(%esp), %edx /* start address */ + movl 20+16(%esp), %ecx /* cpu_has_pae */ + movl 20+20(%esp), %esi /* preserve_context */ + + /* zero out flags, and disable interrupts */ + pushl $0 + popfl + + /* save some information for jumping back */ + movl PTR(VA_CONTROL_PAGE)(%ebp), %edi + movl %edi, CP_VA_CONTROL_PAGE(%edi) + movl PTR(PA_PGD)(%ebp), %eax + movl %eax, CP_PA_PGD(%edi) + movl PTR(PA_SWAP_PAGE)(%ebp), %eax + movl %eax, CP_PA_SWAP_PAGE(%edi) + movl %ebx, CP_PA_BACKUP_PAGES_MAP(%edi) + + /* + * get physical address of control page now + * this is impossible after page table switch + */ + movl PTR(PA_CONTROL_PAGE)(%ebp), %edi + + /* switch to new set of page tables */ + movl PTR(PA_PGD)(%ebp), %eax + movl %eax, %cr3 + + /* setup a new stack at the end of the physical control page */ + lea PAGE_SIZE(%edi), %esp + + /* jump to identity mapped page */ + movl %edi, %eax + addl $(identity_mapped - relocate_kernel), %eax + pushl %eax + ANNOTATE_UNRET_SAFE + ret + int3 +SYM_CODE_END(relocate_kernel) + +SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) + /* set return address to 0 if not preserving context */ + pushl $0 + /* store the start address on the stack */ + pushl %edx + + /* + * Set cr0 to a known state: + * - Paging disabled + * - Alignment check disabled + * - Write protect disabled + * - No task switch + * - Don't do FP software emulation. + * - Protected mode enabled + */ + movl %cr0, %eax + andl $~(X86_CR0_PG | X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %eax + orl $(X86_CR0_PE), %eax + movl %eax, %cr0 + + /* clear cr4 if applicable */ + testl %ecx, %ecx + jz 1f + /* + * Set cr4 to a known state: + * Setting everything to zero seems safe. + */ + xorl %eax, %eax + movl %eax, %cr4 + + jmp 1f +1: + + /* Flush the TLB (needed?) */ + xorl %eax, %eax + movl %eax, %cr3 + + movl CP_PA_SWAP_PAGE(%edi), %eax + pushl %eax + pushl %ebx + call swap_pages + addl $8, %esp + + /* + * To be certain of avoiding problems with self-modifying code + * I need to execute a serializing instruction here. + * So I flush the TLB, it's handy, and not processor dependent. + */ + xorl %eax, %eax + movl %eax, %cr3 + + /* + * set all of the registers to known values + * leave %esp alone + */ + + testl %esi, %esi + jnz 1f + xorl %edi, %edi + xorl %eax, %eax + xorl %ebx, %ebx + xorl %ecx, %ecx + xorl %edx, %edx + xorl %esi, %esi + xorl %ebp, %ebp + ANNOTATE_UNRET_SAFE + ret + int3 +1: + popl %edx + movl CP_PA_SWAP_PAGE(%edi), %esp + addl $PAGE_SIZE, %esp +2: + ANNOTATE_RETPOLINE_SAFE + call *%edx + + /* get the re-entry point of the peer system */ + movl 0(%esp), %ebp + call 1f +1: + popl %ebx + subl $(1b - relocate_kernel), %ebx + movl CP_VA_CONTROL_PAGE(%ebx), %edi + lea PAGE_SIZE(%ebx), %esp + movl CP_PA_SWAP_PAGE(%ebx), %eax + movl CP_PA_BACKUP_PAGES_MAP(%ebx), %edx + pushl %eax + pushl %edx + call swap_pages + addl $8, %esp + movl CP_PA_PGD(%ebx), %eax + movl %eax, %cr3 + movl %cr0, %eax + orl $X86_CR0_PG, %eax + movl %eax, %cr0 + lea PAGE_SIZE(%edi), %esp + movl %edi, %eax + addl $(virtual_mapped - relocate_kernel), %eax + pushl %eax + ANNOTATE_UNRET_SAFE + ret + int3 +SYM_CODE_END(identity_mapped) + +SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) + movl CR4(%edi), %eax + movl %eax, %cr4 + movl CR3(%edi), %eax + movl %eax, %cr3 + movl CR0(%edi), %eax + movl %eax, %cr0 + movl ESP(%edi), %esp + movl %ebp, %eax + + popf + popl %ebp + popl %edi + popl %esi + popl %ebx + ANNOTATE_UNRET_SAFE + ret + int3 +SYM_CODE_END(virtual_mapped) + + /* Do the copies */ +SYM_CODE_START_LOCAL_NOALIGN(swap_pages) + movl 8(%esp), %edx + movl 4(%esp), %ecx + pushl %ebp + pushl %ebx + pushl %edi + pushl %esi + movl %ecx, %ebx + jmp 1f + +0: /* top, read another word from the indirection page */ + movl (%ebx), %ecx + addl $4, %ebx +1: + testb $0x1, %cl /* is it a destination page */ + jz 2f + movl %ecx, %edi + andl $0xfffff000, %edi + jmp 0b +2: + testb $0x2, %cl /* is it an indirection page */ + jz 2f + movl %ecx, %ebx + andl $0xfffff000, %ebx + jmp 0b +2: + testb $0x4, %cl /* is it the done indicator */ + jz 2f + jmp 3f +2: + testb $0x8, %cl /* is it the source indicator */ + jz 0b /* Ignore it otherwise */ + movl %ecx, %esi /* For every source page do a copy */ + andl $0xfffff000, %esi + + movl %edi, %eax + movl %esi, %ebp + + movl %edx, %edi + movl $1024, %ecx + rep ; movsl + + movl %ebp, %edi + movl %eax, %esi + movl $1024, %ecx + rep ; movsl + + movl %eax, %edi + movl %edx, %esi + movl $1024, %ecx + rep ; movsl + + lea PAGE_SIZE(%ebp), %esi + jmp 0b +3: + popl %esi + popl %edi + popl %ebx + popl %ebp + ANNOTATE_UNRET_SAFE + ret + int3 +SYM_CODE_END(swap_pages) + + .globl kexec_control_code_size +.set kexec_control_code_size, . - relocate_kernel |