diff options
Diffstat (limited to '')
| -rw-r--r-- | arch/riscv/kernel/vdso.c | 322 | ||||
| -rw-r--r-- | arch/riscv/kernel/vdso/.gitignore | 4 | ||||
| -rw-r--r-- | arch/riscv/kernel/vdso/Makefile | 83 | ||||
| -rw-r--r-- | arch/riscv/kernel/vdso/flush_icache.S | 22 | ||||
| -rwxr-xr-x | arch/riscv/kernel/vdso/gen_vdso_offsets.sh | 5 | ||||
| -rw-r--r-- | arch/riscv/kernel/vdso/getcpu.S | 18 | ||||
| -rw-r--r-- | arch/riscv/kernel/vdso/note.S | 12 | ||||
| -rw-r--r-- | arch/riscv/kernel/vdso/rt_sigreturn.S | 16 | ||||
| -rw-r--r-- | arch/riscv/kernel/vdso/vdso.S | 23 | ||||
| -rw-r--r-- | arch/riscv/kernel/vdso/vdso.lds.S | 80 | ||||
| -rw-r--r-- | arch/riscv/kernel/vdso/vgettimeofday.c | 31 |
11 files changed, 616 insertions, 0 deletions
diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c new file mode 100644 index 000000000..123d05255 --- /dev/null +++ b/arch/riscv/kernel/vdso.c @@ -0,0 +1,322 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp. + * <benh@kernel.crashing.org> + * Copyright (C) 2012 ARM Limited + * Copyright (C) 2015 Regents of the University of California + */ + +#include <linux/elf.h> +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/binfmts.h> +#include <linux/err.h> +#include <asm/page.h> +#include <asm/vdso.h> +#include <linux/time_namespace.h> + +#ifdef CONFIG_GENERIC_TIME_VSYSCALL +#include <vdso/datapage.h> +#else +struct vdso_data { +}; +#endif + +extern char vdso_start[], vdso_end[]; +#ifdef CONFIG_COMPAT +extern char compat_vdso_start[], compat_vdso_end[]; +#endif + +enum vvar_pages { + VVAR_DATA_PAGE_OFFSET, + VVAR_TIMENS_PAGE_OFFSET, + VVAR_NR_PAGES, +}; + +enum rv_vdso_map { + RV_VDSO_MAP_VVAR, + RV_VDSO_MAP_VDSO, +}; + +#define VVAR_SIZE (VVAR_NR_PAGES << PAGE_SHIFT) + +/* + * The vDSO data page. + */ +static union { + struct vdso_data data; + u8 page[PAGE_SIZE]; +} vdso_data_store __page_aligned_data; +struct vdso_data *vdso_data = &vdso_data_store.data; + +struct __vdso_info { + const char *name; + const char *vdso_code_start; + const char *vdso_code_end; + unsigned long vdso_pages; + /* Data Mapping */ + struct vm_special_mapping *dm; + /* Code Mapping */ + struct vm_special_mapping *cm; +}; + +static struct __vdso_info vdso_info; +#ifdef CONFIG_COMPAT +static struct __vdso_info compat_vdso_info; +#endif + +static int vdso_mremap(const struct vm_special_mapping *sm, + struct vm_area_struct *new_vma) +{ + current->mm->context.vdso = (void *)new_vma->vm_start; + + return 0; +} + +static void __init __vdso_init(struct __vdso_info *vdso_info) +{ + unsigned int i; + struct page **vdso_pagelist; + unsigned long pfn; + + if (memcmp(vdso_info->vdso_code_start, "\177ELF", 4)) + panic("vDSO is not a valid ELF object!\n"); + + vdso_info->vdso_pages = ( + vdso_info->vdso_code_end - + vdso_info->vdso_code_start) >> + PAGE_SHIFT; + + vdso_pagelist = kcalloc(vdso_info->vdso_pages, + sizeof(struct page *), + GFP_KERNEL); + if (vdso_pagelist == NULL) + panic("vDSO kcalloc failed!\n"); + + /* Grab the vDSO code pages. */ + pfn = sym_to_pfn(vdso_info->vdso_code_start); + + for (i = 0; i < vdso_info->vdso_pages; i++) + vdso_pagelist[i] = pfn_to_page(pfn + i); + + vdso_info->cm->pages = vdso_pagelist; +} + +#ifdef CONFIG_TIME_NS +struct vdso_data *arch_get_vdso_data(void *vvar_page) +{ + return (struct vdso_data *)(vvar_page); +} + +/* + * The vvar mapping contains data for a specific time namespace, so when a task + * changes namespace we must unmap its vvar data for the old namespace. + * Subsequent faults will map in data for the new namespace. + * + * For more details see timens_setup_vdso_data(). + */ +int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) +{ + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); + + mmap_read_lock(mm); + + for_each_vma(vmi, vma) { + unsigned long size = vma->vm_end - vma->vm_start; + + if (vma_is_special_mapping(vma, vdso_info.dm)) + zap_page_range(vma, vma->vm_start, size); +#ifdef CONFIG_COMPAT + if (vma_is_special_mapping(vma, compat_vdso_info.dm)) + zap_page_range(vma, vma->vm_start, size); +#endif + } + + mmap_read_unlock(mm); + return 0; +} + +static struct page *find_timens_vvar_page(struct vm_area_struct *vma) +{ + if (likely(vma->vm_mm == current->mm)) + return current->nsproxy->time_ns->vvar_page; + + /* + * VM_PFNMAP | VM_IO protect .fault() handler from being called + * through interfaces like /proc/$pid/mem or + * process_vm_{readv,writev}() as long as there's no .access() + * in special_mapping_vmops. + * For more details check_vma_flags() and __access_remote_vm() + */ + WARN(1, "vvar_page accessed remotely"); + + return NULL; +} +#else +static struct page *find_timens_vvar_page(struct vm_area_struct *vma) +{ + return NULL; +} +#endif + +static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, + struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct page *timens_page = find_timens_vvar_page(vma); + unsigned long pfn; + + switch (vmf->pgoff) { + case VVAR_DATA_PAGE_OFFSET: + if (timens_page) + pfn = page_to_pfn(timens_page); + else + pfn = sym_to_pfn(vdso_data); + break; +#ifdef CONFIG_TIME_NS + case VVAR_TIMENS_PAGE_OFFSET: + /* + * If a task belongs to a time namespace then a namespace + * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and + * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET + * offset. + * See also the comment near timens_setup_vdso_data(). + */ + if (!timens_page) + return VM_FAULT_SIGBUS; + pfn = sym_to_pfn(vdso_data); + break; +#endif /* CONFIG_TIME_NS */ + default: + return VM_FAULT_SIGBUS; + } + + return vmf_insert_pfn(vma, vmf->address, pfn); +} + +static struct vm_special_mapping rv_vdso_maps[] __ro_after_init = { + [RV_VDSO_MAP_VVAR] = { + .name = "[vvar]", + .fault = vvar_fault, + }, + [RV_VDSO_MAP_VDSO] = { + .name = "[vdso]", + .mremap = vdso_mremap, + }, +}; + +static struct __vdso_info vdso_info __ro_after_init = { + .name = "vdso", + .vdso_code_start = vdso_start, + .vdso_code_end = vdso_end, + .dm = &rv_vdso_maps[RV_VDSO_MAP_VVAR], + .cm = &rv_vdso_maps[RV_VDSO_MAP_VDSO], +}; + +#ifdef CONFIG_COMPAT +static struct vm_special_mapping rv_compat_vdso_maps[] __ro_after_init = { + [RV_VDSO_MAP_VVAR] = { + .name = "[vvar]", + .fault = vvar_fault, + }, + [RV_VDSO_MAP_VDSO] = { + .name = "[vdso]", + .mremap = vdso_mremap, + }, +}; + +static struct __vdso_info compat_vdso_info __ro_after_init = { + .name = "compat_vdso", + .vdso_code_start = compat_vdso_start, + .vdso_code_end = compat_vdso_end, + .dm = &rv_compat_vdso_maps[RV_VDSO_MAP_VVAR], + .cm = &rv_compat_vdso_maps[RV_VDSO_MAP_VDSO], +}; +#endif + +static int __init vdso_init(void) +{ + __vdso_init(&vdso_info); +#ifdef CONFIG_COMPAT + __vdso_init(&compat_vdso_info); +#endif + + return 0; +} +arch_initcall(vdso_init); + +static int __setup_additional_pages(struct mm_struct *mm, + struct linux_binprm *bprm, + int uses_interp, + struct __vdso_info *vdso_info) +{ + unsigned long vdso_base, vdso_text_len, vdso_mapping_len; + void *ret; + + BUILD_BUG_ON(VVAR_NR_PAGES != __VVAR_PAGES); + + vdso_text_len = vdso_info->vdso_pages << PAGE_SHIFT; + /* Be sure to map the data page */ + vdso_mapping_len = vdso_text_len + VVAR_SIZE; + + vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0); + if (IS_ERR_VALUE(vdso_base)) { + ret = ERR_PTR(vdso_base); + goto up_fail; + } + + ret = _install_special_mapping(mm, vdso_base, VVAR_SIZE, + (VM_READ | VM_MAYREAD | VM_PFNMAP), vdso_info->dm); + if (IS_ERR(ret)) + goto up_fail; + + vdso_base += VVAR_SIZE; + mm->context.vdso = (void *)vdso_base; + + ret = + _install_special_mapping(mm, vdso_base, vdso_text_len, + (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC), + vdso_info->cm); + + if (IS_ERR(ret)) + goto up_fail; + + return 0; + +up_fail: + mm->context.vdso = NULL; + return PTR_ERR(ret); +} + +#ifdef CONFIG_COMPAT +int compat_arch_setup_additional_pages(struct linux_binprm *bprm, + int uses_interp) +{ + struct mm_struct *mm = current->mm; + int ret; + + if (mmap_write_lock_killable(mm)) + return -EINTR; + + ret = __setup_additional_pages(mm, bprm, uses_interp, + &compat_vdso_info); + mmap_write_unlock(mm); + + return ret; +} +#endif + +int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) +{ + struct mm_struct *mm = current->mm; + int ret; + + if (mmap_write_lock_killable(mm)) + return -EINTR; + + ret = __setup_additional_pages(mm, bprm, uses_interp, &vdso_info); + mmap_write_unlock(mm); + + return ret; +} diff --git a/arch/riscv/kernel/vdso/.gitignore b/arch/riscv/kernel/vdso/.gitignore new file mode 100644 index 000000000..3a19def86 --- /dev/null +++ b/arch/riscv/kernel/vdso/.gitignore @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +vdso.lds +*.tmp +vdso-syms.S diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile new file mode 100644 index 000000000..06e6b27f3 --- /dev/null +++ b/arch/riscv/kernel/vdso/Makefile @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copied from arch/tile/kernel/vdso/Makefile + +# Absolute relocation type $(ARCH_REL_TYPE_ABS) needs to be defined before +# the inclusion of generic Makefile. +ARCH_REL_TYPE_ABS := R_RISCV_32|R_RISCV_64|R_RISCV_JUMP_SLOT +include $(srctree)/lib/vdso/Makefile +# Symbols present in the vdso +vdso-syms = rt_sigreturn +ifdef CONFIG_64BIT +vdso-syms += vgettimeofday +endif +vdso-syms += getcpu +vdso-syms += flush_icache + +# Files to link into the vdso +obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o + +ccflags-y := -fno-stack-protector +ccflags-y += -DDISABLE_BRANCH_PROFILING + +ifneq ($(c-gettimeofday-y),) + CFLAGS_vgettimeofday.o += -fPIC -include $(c-gettimeofday-y) +endif + +# Build rules +targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds +obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) + +obj-y += vdso.o +CPPFLAGS_vdso.lds += -P -C -U$(ARCH) +ifneq ($(filter vgettimeofday, $(vdso-syms)),) +CPPFLAGS_vdso.lds += -DHAS_VGETTIMEOFDAY +endif + +# Disable -pg to prevent insert call site +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) + +# Disable profiling and instrumentation for VDSO code +GCOV_PROFILE := n +KCOV_INSTRUMENT := n +KASAN_SANITIZE := n +UBSAN_SANITIZE := n + +# Force dependency +$(obj)/vdso.o: $(obj)/vdso.so + +# link rule for the .so file, .lds has to be first +$(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) FORCE + $(call if_changed,vdsold) +LDFLAGS_vdso.so.dbg = -shared -S -soname=linux-vdso.so.1 \ + --build-id=sha1 --hash-style=both --eh-frame-hdr + +# strip rule for the .so file +$(obj)/%.so: OBJCOPYFLAGS := -S +$(obj)/%.so: $(obj)/%.so.dbg FORCE + $(call if_changed,objcopy) + +# Generate VDSO offsets using helper script +gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh +quiet_cmd_vdsosym = VDSOSYM $@ + cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ + +include/generated/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE + $(call if_changed,vdsosym) + +# actual build commands +# The DSO images are built using a special linker script +# Make sure only to export the intended __vdso_xxx symbol offsets. +quiet_cmd_vdsold = VDSOLD $@ + cmd_vdsold = $(LD) $(ld_flags) -T $(filter-out FORCE,$^) -o $@.tmp && \ + $(OBJCOPY) $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \ + rm $@.tmp + +# install commands for the unstripped file +quiet_cmd_vdso_install = INSTALL $@ + cmd_vdso_install = cp $(obj)/$@.dbg $(MODLIB)/vdso/$@ + +vdso.so: $(obj)/vdso.so.dbg + @mkdir -p $(MODLIB)/vdso + $(call cmd,vdso_install) + +vdso_install: vdso.so diff --git a/arch/riscv/kernel/vdso/flush_icache.S b/arch/riscv/kernel/vdso/flush_icache.S new file mode 100644 index 000000000..82f97d67c --- /dev/null +++ b/arch/riscv/kernel/vdso/flush_icache.S @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017 SiFive + */ + +#include <linux/linkage.h> +#include <asm/unistd.h> + + .text +/* int __vdso_flush_icache(void *start, void *end, unsigned long flags); */ +ENTRY(__vdso_flush_icache) + .cfi_startproc +#ifdef CONFIG_SMP + li a7, __NR_riscv_flush_icache + ecall +#else + fence.i + li a0, 0 +#endif + ret + .cfi_endproc +ENDPROC(__vdso_flush_icache) diff --git a/arch/riscv/kernel/vdso/gen_vdso_offsets.sh b/arch/riscv/kernel/vdso/gen_vdso_offsets.sh new file mode 100755 index 000000000..c2e5613f3 --- /dev/null +++ b/arch/riscv/kernel/vdso/gen_vdso_offsets.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +LC_ALL=C +sed -n -e 's/^[0]\+\(0[0-9a-fA-F]*\) . \(__vdso_[a-zA-Z0-9_]*\)$/\#define \2_offset\t0x\1/p' diff --git a/arch/riscv/kernel/vdso/getcpu.S b/arch/riscv/kernel/vdso/getcpu.S new file mode 100644 index 000000000..bb0c05e2f --- /dev/null +++ b/arch/riscv/kernel/vdso/getcpu.S @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017 SiFive + */ + +#include <linux/linkage.h> +#include <asm/unistd.h> + + .text +/* int __vdso_getcpu(unsigned *cpu, unsigned *node, void *unused); */ +ENTRY(__vdso_getcpu) + .cfi_startproc + /* For now, just do the syscall. */ + li a7, __NR_getcpu + ecall + ret + .cfi_endproc +ENDPROC(__vdso_getcpu) diff --git a/arch/riscv/kernel/vdso/note.S b/arch/riscv/kernel/vdso/note.S new file mode 100644 index 000000000..2a956c942 --- /dev/null +++ b/arch/riscv/kernel/vdso/note.S @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * This supplies .note.* sections to go into the PT_NOTE inside the vDSO text. + * Here we can supply some information useful to userland. + */ + +#include <linux/elfnote.h> +#include <linux/version.h> + +ELFNOTE_START(Linux, 0, "a") + .long LINUX_VERSION_CODE +ELFNOTE_END diff --git a/arch/riscv/kernel/vdso/rt_sigreturn.S b/arch/riscv/kernel/vdso/rt_sigreturn.S new file mode 100644 index 000000000..0573705ea --- /dev/null +++ b/arch/riscv/kernel/vdso/rt_sigreturn.S @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2014 Regents of the University of California + */ + +#include <linux/linkage.h> +#include <asm/unistd.h> + + .text +ENTRY(__vdso_rt_sigreturn) + .cfi_startproc + .cfi_signal_frame + li a7, __NR_rt_sigreturn + scall + .cfi_endproc +ENDPROC(__vdso_rt_sigreturn) diff --git a/arch/riscv/kernel/vdso/vdso.S b/arch/riscv/kernel/vdso/vdso.S new file mode 100644 index 000000000..83f1c899e --- /dev/null +++ b/arch/riscv/kernel/vdso/vdso.S @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2014 Regents of the University of California + */ + +#include <linux/init.h> +#include <linux/linkage.h> +#include <asm/page.h> + +#ifndef __VDSO_PATH +#define __VDSO_PATH "arch/riscv/kernel/vdso/vdso.so" +#endif + + __PAGE_ALIGNED_DATA + + .globl vdso_start, vdso_end + .balign PAGE_SIZE +vdso_start: + .incbin __VDSO_PATH + .balign PAGE_SIZE +vdso_end: + + .previous diff --git a/arch/riscv/kernel/vdso/vdso.lds.S b/arch/riscv/kernel/vdso/vdso.lds.S new file mode 100644 index 000000000..150b1a572 --- /dev/null +++ b/arch/riscv/kernel/vdso/vdso.lds.S @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Regents of the University of California + */ +#include <asm/page.h> +#include <asm/vdso.h> + +OUTPUT_ARCH(riscv) + +SECTIONS +{ + PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE); +#ifdef CONFIG_TIME_NS + PROVIDE(_timens_data = _vdso_data + PAGE_SIZE); +#endif + . = SIZEOF_HEADERS; + + .hash : { *(.hash) } :text + .gnu.hash : { *(.gnu.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .gnu.version : { *(.gnu.version) } + .gnu.version_d : { *(.gnu.version_d) } + .gnu.version_r : { *(.gnu.version_r) } + + .note : { *(.note.*) } :text :note + .dynamic : { *(.dynamic) } :text :dynamic + + .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr + .eh_frame : { KEEP (*(.eh_frame)) } :text + + .rodata : { *(.rodata .rodata.* .gnu.linkonce.r.*) } + + /* + * This linker script is used both with -r and with -shared. + * For the layouts to match, we need to skip more than enough + * space for the dynamic symbol table, etc. If this amount is + * insufficient, ld -shared will error; simply increase it here. + */ + . = 0x800; + .text : { *(.text .text.*) } :text + + .data : { + *(.got.plt) *(.got) + *(.data .data.* .gnu.linkonce.d.*) + *(.dynbss) + *(.bss .bss.* .gnu.linkonce.b.*) + } +} + +/* + * We must supply the ELF program headers explicitly to get just one + * PT_LOAD segment, and set the flags explicitly to make segments read-only. + */ +PHDRS +{ + text PT_LOAD FLAGS(5) FILEHDR PHDRS; /* PF_R|PF_X */ + dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ + note PT_NOTE FLAGS(4); /* PF_R */ + eh_frame_hdr PT_GNU_EH_FRAME; +} + +/* + * This controls what symbols we export from the DSO. + */ +VERSION +{ + LINUX_4.15 { + global: + __vdso_rt_sigreturn; +#ifdef HAS_VGETTIMEOFDAY + __vdso_gettimeofday; + __vdso_clock_gettime; + __vdso_clock_getres; +#endif + __vdso_getcpu; + __vdso_flush_icache; + local: *; + }; +} diff --git a/arch/riscv/kernel/vdso/vgettimeofday.c b/arch/riscv/kernel/vdso/vgettimeofday.c new file mode 100644 index 000000000..cc0d80699 --- /dev/null +++ b/arch/riscv/kernel/vdso/vgettimeofday.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copied from arch/arm64/kernel/vdso/vgettimeofday.c + * + * Copyright (C) 2018 ARM Ltd. + * Copyright (C) 2020 SiFive + */ + +#include <linux/time.h> +#include <linux/types.h> + +extern +int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts); +int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime(clock, ts); +} + +extern +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz); +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + return __cvdso_gettimeofday(tv, tz); +} + +extern +int __vdso_clock_getres(clockid_t clock_id, struct __kernel_timespec *res); +int __vdso_clock_getres(clockid_t clock_id, struct __kernel_timespec *res) +{ + return __cvdso_clock_getres(clock_id, res); +} |