diff options
Diffstat (limited to 'arch/x86/mm/mem_encrypt_boot.S')
| -rw-r--r-- | arch/x86/mm/mem_encrypt_boot.S | 162 |
1 files changed, 162 insertions, 0 deletions
diff --git a/arch/x86/mm/mem_encrypt_boot.S b/arch/x86/mm/mem_encrypt_boot.S new file mode 100644 index 000000000..9de3d900b --- /dev/null +++ b/arch/x86/mm/mem_encrypt_boot.S @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AMD Memory Encryption Support + * + * Copyright (C) 2016 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + */ + +#include <linux/linkage.h> +#include <linux/pgtable.h> +#include <asm/page.h> +#include <asm/processor-flags.h> +#include <asm/msr-index.h> +#include <asm/nospec-branch.h> + + .text + .code64 +SYM_FUNC_START(sme_encrypt_execute) + + /* + * Entry parameters: + * RDI - virtual address for the encrypted mapping + * RSI - virtual address for the decrypted mapping + * RDX - length to encrypt + * RCX - virtual address of the encryption workarea, including: + * - stack page (PAGE_SIZE) + * - encryption routine page (PAGE_SIZE) + * - intermediate copy buffer (PMD_PAGE_SIZE) + * R8 - physical address of the pagetables to use for encryption + */ + + push %rbp + movq %rsp, %rbp /* RBP now has original stack pointer */ + + /* Set up a one page stack in the non-encrypted memory area */ + movq %rcx, %rax /* Workarea stack page */ + leaq PAGE_SIZE(%rax), %rsp /* Set new stack pointer */ + addq $PAGE_SIZE, %rax /* Workarea encryption routine */ + + push %r12 + movq %rdi, %r10 /* Encrypted area */ + movq %rsi, %r11 /* Decrypted area */ + movq %rdx, %r12 /* Area length */ + + /* Copy encryption routine into the workarea */ + movq %rax, %rdi /* Workarea encryption routine */ + leaq __enc_copy(%rip), %rsi /* Encryption routine */ + movq $(.L__enc_copy_end - __enc_copy), %rcx /* Encryption routine length */ + rep movsb + + /* Setup registers for call */ + movq %r10, %rdi /* Encrypted area */ + movq %r11, %rsi /* Decrypted area */ + movq %r8, %rdx /* Pagetables used for encryption */ + movq %r12, %rcx /* Area length */ + movq %rax, %r8 /* Workarea encryption routine */ + addq $PAGE_SIZE, %r8 /* Workarea intermediate copy buffer */ + + ANNOTATE_RETPOLINE_SAFE + call *%rax /* Call the encryption routine */ + + pop %r12 + + movq %rbp, %rsp /* Restore original stack pointer */ + pop %rbp + + /* Offset to __x86_return_thunk would be wrong here */ + ANNOTATE_UNRET_SAFE + ret + int3 +SYM_FUNC_END(sme_encrypt_execute) + +SYM_FUNC_START(__enc_copy) +/* + * Routine used to encrypt memory in place. + * This routine must be run outside of the kernel proper since + * the kernel will be encrypted during the process. So this + * routine is defined here and then copied to an area outside + * of the kernel where it will remain and run decrypted + * during execution. + * + * On entry the registers must be: + * RDI - virtual address for the encrypted mapping + * RSI - virtual address for the decrypted mapping + * RDX - address of the pagetables to use for encryption + * RCX - length of area + * R8 - intermediate copy buffer + * + * RAX - points to this routine + * + * The area will be encrypted by copying from the non-encrypted + * memory space to an intermediate buffer and then copying from the + * intermediate buffer back to the encrypted memory space. The physical + * addresses of the two mappings are the same which results in the area + * being encrypted "in place". + */ + /* Enable the new page tables */ + mov %rdx, %cr3 + + /* Flush any global TLBs */ + mov %cr4, %rdx + andq $~X86_CR4_PGE, %rdx + mov %rdx, %cr4 + orq $X86_CR4_PGE, %rdx + mov %rdx, %cr4 + + push %r15 + push %r12 + + movq %rcx, %r9 /* Save area length */ + movq %rdi, %r10 /* Save encrypted area address */ + movq %rsi, %r11 /* Save decrypted area address */ + + /* Set the PAT register PA5 entry to write-protect */ + movl $MSR_IA32_CR_PAT, %ecx + rdmsr + mov %rdx, %r15 /* Save original PAT value */ + andl $0xffff00ff, %edx /* Clear PA5 */ + orl $0x00000500, %edx /* Set PA5 to WP */ + wrmsr + + wbinvd /* Invalidate any cache entries */ + + /* Copy/encrypt up to 2MB at a time */ + movq $PMD_PAGE_SIZE, %r12 +1: + cmpq %r12, %r9 + jnb 2f + movq %r9, %r12 + +2: + movq %r11, %rsi /* Source - decrypted area */ + movq %r8, %rdi /* Dest - intermediate copy buffer */ + movq %r12, %rcx + rep movsb + + movq %r8, %rsi /* Source - intermediate copy buffer */ + movq %r10, %rdi /* Dest - encrypted area */ + movq %r12, %rcx + rep movsb + + addq %r12, %r11 + addq %r12, %r10 + subq %r12, %r9 /* Kernel length decrement */ + jnz 1b /* Kernel length not zero? */ + + /* Restore PAT register */ + movl $MSR_IA32_CR_PAT, %ecx + rdmsr + mov %r15, %rdx /* Restore original PAT value */ + wrmsr + + pop %r12 + pop %r15 + + /* Offset to __x86_return_thunk would be wrong here */ + ANNOTATE_UNRET_SAFE + ret + int3 +.L__enc_copy_end: +SYM_FUNC_END(__enc_copy) |