diff options
Diffstat (limited to 'arch/xtensa/mm')
| -rw-r--r-- | arch/xtensa/mm/Makefile | 12 | ||||
| -rw-r--r-- | arch/xtensa/mm/cache.c | 328 | ||||
| -rw-r--r-- | arch/xtensa/mm/fault.c | 255 | ||||
| -rw-r--r-- | arch/xtensa/mm/highmem.c | 95 | ||||
| -rw-r--r-- | arch/xtensa/mm/init.c | 269 | ||||
| -rw-r--r-- | arch/xtensa/mm/ioremap.c | 68 | ||||
| -rw-r--r-- | arch/xtensa/mm/kasan_init.c | 95 | ||||
| -rw-r--r-- | arch/xtensa/mm/misc.S | 484 | ||||
| -rw-r--r-- | arch/xtensa/mm/mmu.c | 114 | ||||
| -rw-r--r-- | arch/xtensa/mm/tlb.c | 276 |
10 files changed, 1996 insertions, 0 deletions
diff --git a/arch/xtensa/mm/Makefile b/arch/xtensa/mm/Makefile new file mode 100644 index 000000000..734888a00 --- /dev/null +++ b/arch/xtensa/mm/Makefile @@ -0,0 +1,12 @@ +# +# Makefile for the Linux/Xtensa-specific parts of the memory manager. +# + +obj-y := init.o misc.o +obj-$(CONFIG_MMU) += cache.o fault.o ioremap.o mmu.o tlb.o +obj-$(CONFIG_HIGHMEM) += highmem.o +obj-$(CONFIG_KASAN) += kasan_init.o + +KASAN_SANITIZE_fault.o := n +KASAN_SANITIZE_kasan_init.o := n +KASAN_SANITIZE_mmu.o := n diff --git a/arch/xtensa/mm/cache.c b/arch/xtensa/mm/cache.c new file mode 100644 index 000000000..d1ebe67c6 --- /dev/null +++ b/arch/xtensa/mm/cache.c @@ -0,0 +1,328 @@ +/* + * arch/xtensa/mm/cache.c + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2001-2006 Tensilica Inc. + * + * Chris Zankel <chris@zankel.net> + * Joe Taylor + * Marc Gauthier + * + */ + +#include <linux/init.h> +#include <linux/signal.h> +#include <linux/sched.h> +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/string.h> +#include <linux/types.h> +#include <linux/ptrace.h> +#include <linux/bootmem.h> +#include <linux/swap.h> +#include <linux/pagemap.h> + +#include <asm/bootparam.h> +#include <asm/mmu_context.h> +#include <asm/tlb.h> +#include <asm/tlbflush.h> +#include <asm/page.h> +#include <asm/pgalloc.h> +#include <asm/pgtable.h> + +/* + * Note: + * The kernel provides one architecture bit PG_arch_1 in the page flags that + * can be used for cache coherency. + * + * I$-D$ coherency. + * + * The Xtensa architecture doesn't keep the instruction cache coherent with + * the data cache. We use the architecture bit to indicate if the caches + * are coherent. The kernel clears this bit whenever a page is added to the + * page cache. At that time, the caches might not be in sync. We, therefore, + * define this flag as 'clean' if set. + * + * D-cache aliasing. + * + * With cache aliasing, we have to always flush the cache when pages are + * unmapped (see tlb_start_vma(). So, we use this flag to indicate a dirty + * page. + * + * + * + */ + +#if (DCACHE_WAY_SIZE > PAGE_SIZE) +static inline void kmap_invalidate_coherent(struct page *page, + unsigned long vaddr) +{ + if (!DCACHE_ALIAS_EQ(page_to_phys(page), vaddr)) { + unsigned long kvaddr; + + if (!PageHighMem(page)) { + kvaddr = (unsigned long)page_to_virt(page); + + __invalidate_dcache_page(kvaddr); + } else { + kvaddr = TLBTEMP_BASE_1 + + (page_to_phys(page) & DCACHE_ALIAS_MASK); + + preempt_disable(); + __invalidate_dcache_page_alias(kvaddr, + page_to_phys(page)); + preempt_enable(); + } + } +} + +static inline void *coherent_kvaddr(struct page *page, unsigned long base, + unsigned long vaddr, unsigned long *paddr) +{ + if (PageHighMem(page) || !DCACHE_ALIAS_EQ(page_to_phys(page), vaddr)) { + *paddr = page_to_phys(page); + return (void *)(base + (vaddr & DCACHE_ALIAS_MASK)); + } else { + *paddr = 0; + return page_to_virt(page); + } +} + +void clear_user_highpage(struct page *page, unsigned long vaddr) +{ + unsigned long paddr; + void *kvaddr = coherent_kvaddr(page, TLBTEMP_BASE_1, vaddr, &paddr); + + preempt_disable(); + kmap_invalidate_coherent(page, vaddr); + set_bit(PG_arch_1, &page->flags); + clear_page_alias(kvaddr, paddr); + preempt_enable(); +} +EXPORT_SYMBOL(clear_user_highpage); + +void copy_user_highpage(struct page *dst, struct page *src, + unsigned long vaddr, struct vm_area_struct *vma) +{ + unsigned long dst_paddr, src_paddr; + void *dst_vaddr = coherent_kvaddr(dst, TLBTEMP_BASE_1, vaddr, + &dst_paddr); + void *src_vaddr = coherent_kvaddr(src, TLBTEMP_BASE_2, vaddr, + &src_paddr); + + preempt_disable(); + kmap_invalidate_coherent(dst, vaddr); + set_bit(PG_arch_1, &dst->flags); + copy_page_alias(dst_vaddr, src_vaddr, dst_paddr, src_paddr); + preempt_enable(); +} +EXPORT_SYMBOL(copy_user_highpage); + +/* + * Any time the kernel writes to a user page cache page, or it is about to + * read from a page cache page this routine is called. + * + */ + +void flush_dcache_page(struct page *page) +{ + struct address_space *mapping = page_mapping_file(page); + + /* + * If we have a mapping but the page is not mapped to user-space + * yet, we simply mark this page dirty and defer flushing the + * caches until update_mmu(). + */ + + if (mapping && !mapping_mapped(mapping)) { + if (!test_bit(PG_arch_1, &page->flags)) + set_bit(PG_arch_1, &page->flags); + return; + + } else { + + unsigned long phys = page_to_phys(page); + unsigned long temp = page->index << PAGE_SHIFT; + unsigned long alias = !(DCACHE_ALIAS_EQ(temp, phys)); + unsigned long virt; + + /* + * Flush the page in kernel space and user space. + * Note that we can omit that step if aliasing is not + * an issue, but we do have to synchronize I$ and D$ + * if we have a mapping. + */ + + if (!alias && !mapping) + return; + + preempt_disable(); + virt = TLBTEMP_BASE_1 + (phys & DCACHE_ALIAS_MASK); + __flush_invalidate_dcache_page_alias(virt, phys); + + virt = TLBTEMP_BASE_1 + (temp & DCACHE_ALIAS_MASK); + + if (alias) + __flush_invalidate_dcache_page_alias(virt, phys); + + if (mapping) + __invalidate_icache_page_alias(virt, phys); + preempt_enable(); + } + + /* There shouldn't be an entry in the cache for this page anymore. */ +} +EXPORT_SYMBOL(flush_dcache_page); + +/* + * For now, flush the whole cache. FIXME?? + */ + +void local_flush_cache_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + __flush_invalidate_dcache_all(); + __invalidate_icache_all(); +} +EXPORT_SYMBOL(local_flush_cache_range); + +/* + * Remove any entry in the cache for this page. + * + * Note that this function is only called for user pages, so use the + * alias versions of the cache flush functions. + */ + +void local_flush_cache_page(struct vm_area_struct *vma, unsigned long address, + unsigned long pfn) +{ + /* Note that we have to use the 'alias' address to avoid multi-hit */ + + unsigned long phys = page_to_phys(pfn_to_page(pfn)); + unsigned long virt = TLBTEMP_BASE_1 + (address & DCACHE_ALIAS_MASK); + + preempt_disable(); + __flush_invalidate_dcache_page_alias(virt, phys); + __invalidate_icache_page_alias(virt, phys); + preempt_enable(); +} +EXPORT_SYMBOL(local_flush_cache_page); + +#endif /* DCACHE_WAY_SIZE > PAGE_SIZE */ + +void +update_mmu_cache(struct vm_area_struct * vma, unsigned long addr, pte_t *ptep) +{ + unsigned long pfn = pte_pfn(*ptep); + struct page *page; + + if (!pfn_valid(pfn)) + return; + + page = pfn_to_page(pfn); + + /* Invalidate old entry in TLBs */ + + flush_tlb_page(vma, addr); + +#if (DCACHE_WAY_SIZE > PAGE_SIZE) + + if (!PageReserved(page) && test_bit(PG_arch_1, &page->flags)) { + unsigned long phys = page_to_phys(page); + unsigned long tmp; + + preempt_disable(); + tmp = TLBTEMP_BASE_1 + (phys & DCACHE_ALIAS_MASK); + __flush_invalidate_dcache_page_alias(tmp, phys); + tmp = TLBTEMP_BASE_1 + (addr & DCACHE_ALIAS_MASK); + __flush_invalidate_dcache_page_alias(tmp, phys); + __invalidate_icache_page_alias(tmp, phys); + preempt_enable(); + + clear_bit(PG_arch_1, &page->flags); + } +#else + if (!PageReserved(page) && !test_bit(PG_arch_1, &page->flags) + && (vma->vm_flags & VM_EXEC) != 0) { + unsigned long paddr = (unsigned long)kmap_atomic(page); + __flush_dcache_page(paddr); + __invalidate_icache_page(paddr); + set_bit(PG_arch_1, &page->flags); + kunmap_atomic((void *)paddr); + } +#endif +} + +/* + * access_process_vm() has called get_user_pages(), which has done a + * flush_dcache_page() on the page. + */ + +#if (DCACHE_WAY_SIZE > PAGE_SIZE) + +void copy_to_user_page(struct vm_area_struct *vma, struct page *page, + unsigned long vaddr, void *dst, const void *src, + unsigned long len) +{ + unsigned long phys = page_to_phys(page); + unsigned long alias = !(DCACHE_ALIAS_EQ(vaddr, phys)); + + /* Flush and invalidate user page if aliased. */ + + if (alias) { + unsigned long t = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK); + preempt_disable(); + __flush_invalidate_dcache_page_alias(t, phys); + preempt_enable(); + } + + /* Copy data */ + + memcpy(dst, src, len); + + /* + * Flush and invalidate kernel page if aliased and synchronize + * data and instruction caches for executable pages. + */ + + if (alias) { + unsigned long t = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK); + + preempt_disable(); + __flush_invalidate_dcache_range((unsigned long) dst, len); + if ((vma->vm_flags & VM_EXEC) != 0) + __invalidate_icache_page_alias(t, phys); + preempt_enable(); + + } else if ((vma->vm_flags & VM_EXEC) != 0) { + __flush_dcache_range((unsigned long)dst,len); + __invalidate_icache_range((unsigned long) dst, len); + } +} + +extern void copy_from_user_page(struct vm_area_struct *vma, struct page *page, + unsigned long vaddr, void *dst, const void *src, + unsigned long len) +{ + unsigned long phys = page_to_phys(page); + unsigned long alias = !(DCACHE_ALIAS_EQ(vaddr, phys)); + + /* + * Flush user page if aliased. + * (Note: a simply flush would be sufficient) + */ + + if (alias) { + unsigned long t = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK); + preempt_disable(); + __flush_invalidate_dcache_page_alias(t, phys); + preempt_enable(); + } + + memcpy(dst, src, len); +} + +#endif diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c new file mode 100644 index 000000000..2ab0e0dcd --- /dev/null +++ b/arch/xtensa/mm/fault.c @@ -0,0 +1,255 @@ +// TODO VM_EXEC flag work-around, cache aliasing +/* + * arch/xtensa/mm/fault.c + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2001 - 2010 Tensilica Inc. + * + * Chris Zankel <chris@zankel.net> + * Joe Taylor <joe@tensilica.com, joetylr@yahoo.com> + */ + +#include <linux/mm.h> +#include <linux/extable.h> +#include <linux/hardirq.h> +#include <linux/perf_event.h> +#include <linux/uaccess.h> +#include <asm/mmu_context.h> +#include <asm/cacheflush.h> +#include <asm/hardirq.h> +#include <asm/pgalloc.h> + +DEFINE_PER_CPU(unsigned long, asid_cache) = ASID_USER_FIRST; +void bad_page_fault(struct pt_regs*, unsigned long, int); + +/* + * This routine handles page faults. It determines the address, + * and the problem, and then passes it off to one of the appropriate + * routines. + * + * Note: does not handle Miss and MultiHit. + */ + +void do_page_fault(struct pt_regs *regs) +{ + struct vm_area_struct * vma; + struct mm_struct *mm = current->mm; + unsigned int exccause = regs->exccause; + unsigned int address = regs->excvaddr; + int code; + + int is_write, is_exec; + vm_fault_t fault; + unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + + code = SEGV_MAPERR; + + /* We fault-in kernel-space virtual memory on-demand. The + * 'reference' page table is init_mm.pgd. + */ + if (address >= TASK_SIZE && !user_mode(regs)) + goto vmalloc_fault; + + /* If we're in an interrupt or have no user + * context, we must not take the fault.. + */ + if (faulthandler_disabled() || !mm) { + bad_page_fault(regs, address, SIGSEGV); + return; + } + + is_write = (exccause == EXCCAUSE_STORE_CACHE_ATTRIBUTE) ? 1 : 0; + is_exec = (exccause == EXCCAUSE_ITLB_PRIVILEGE || + exccause == EXCCAUSE_ITLB_MISS || + exccause == EXCCAUSE_FETCH_CACHE_ATTRIBUTE) ? 1 : 0; + + pr_debug("[%s:%d:%08x:%d:%08lx:%s%s]\n", + current->comm, current->pid, + address, exccause, regs->pc, + is_write ? "w" : "", is_exec ? "x" : ""); + + if (user_mode(regs)) + flags |= FAULT_FLAG_USER; +retry: + down_read(&mm->mmap_sem); + vma = find_vma(mm, address); + + if (!vma) + goto bad_area; + if (vma->vm_start <= address) + goto good_area; + if (!(vma->vm_flags & VM_GROWSDOWN)) + goto bad_area; + if (expand_stack(vma, address)) + goto bad_area; + + /* Ok, we have a good vm_area for this memory access, so + * we can handle it.. + */ + +good_area: + code = SEGV_ACCERR; + + if (is_write) { + if (!(vma->vm_flags & VM_WRITE)) + goto bad_area; + flags |= FAULT_FLAG_WRITE; + } else if (is_exec) { + if (!(vma->vm_flags & VM_EXEC)) + goto bad_area; + } else /* Allow read even from write-only pages. */ + if (!(vma->vm_flags & (VM_READ | VM_WRITE))) + goto bad_area; + + /* If for any reason at all we couldn't handle the fault, + * make sure we exit gracefully rather than endlessly redo + * the fault. + */ + fault = handle_mm_fault(vma, address, flags); + + if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + return; + + if (unlikely(fault & VM_FAULT_ERROR)) { + if (fault & VM_FAULT_OOM) + goto out_of_memory; + else if (fault & VM_FAULT_SIGSEGV) + goto bad_area; + else if (fault & VM_FAULT_SIGBUS) + goto do_sigbus; + BUG(); + } + if (flags & FAULT_FLAG_ALLOW_RETRY) { + if (fault & VM_FAULT_MAJOR) + current->maj_flt++; + else + current->min_flt++; + if (fault & VM_FAULT_RETRY) { + flags &= ~FAULT_FLAG_ALLOW_RETRY; + flags |= FAULT_FLAG_TRIED; + + /* No need to up_read(&mm->mmap_sem) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + + goto retry; + } + } + + up_read(&mm->mmap_sem); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); + if (flags & VM_FAULT_MAJOR) + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); + else + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); + + return; + + /* Something tried to access memory that isn't in our memory map.. + * Fix it, but check if it's kernel or user first.. + */ +bad_area: + up_read(&mm->mmap_sem); + if (user_mode(regs)) { + current->thread.bad_vaddr = address; + current->thread.error_code = is_write; + force_sig_fault(SIGSEGV, code, (void *) address, current); + return; + } + bad_page_fault(regs, address, SIGSEGV); + return; + + + /* We ran out of memory, or some other thing happened to us that made + * us unable to handle the page fault gracefully. + */ +out_of_memory: + up_read(&mm->mmap_sem); + if (!user_mode(regs)) + bad_page_fault(regs, address, SIGKILL); + else + pagefault_out_of_memory(); + return; + +do_sigbus: + up_read(&mm->mmap_sem); + + /* Send a sigbus, regardless of whether we were in kernel + * or user mode. + */ + current->thread.bad_vaddr = address; + force_sig_fault(SIGBUS, BUS_ADRERR, (void *) address, current); + + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) + bad_page_fault(regs, address, SIGBUS); + return; + +vmalloc_fault: + { + /* Synchronize this task's top level page-table + * with the 'reference' page table. + */ + struct mm_struct *act_mm = current->active_mm; + int index = pgd_index(address); + pgd_t *pgd, *pgd_k; + pmd_t *pmd, *pmd_k; + pte_t *pte_k; + + if (act_mm == NULL) + goto bad_page_fault; + + pgd = act_mm->pgd + index; + pgd_k = init_mm.pgd + index; + + if (!pgd_present(*pgd_k)) + goto bad_page_fault; + + pgd_val(*pgd) = pgd_val(*pgd_k); + + pmd = pmd_offset(pgd, address); + pmd_k = pmd_offset(pgd_k, address); + if (!pmd_present(*pmd) || !pmd_present(*pmd_k)) + goto bad_page_fault; + + pmd_val(*pmd) = pmd_val(*pmd_k); + pte_k = pte_offset_kernel(pmd_k, address); + + if (!pte_present(*pte_k)) + goto bad_page_fault; + return; + } +bad_page_fault: + bad_page_fault(regs, address, SIGKILL); + return; +} + + +void +bad_page_fault(struct pt_regs *regs, unsigned long address, int sig) +{ + extern void die(const char*, struct pt_regs*, long); + const struct exception_table_entry *entry; + + /* Are we prepared to handle this kernel fault? */ + if ((entry = search_exception_tables(regs->pc)) != NULL) { + pr_debug("%s: Exception at pc=%#010lx (%lx)\n", + current->comm, regs->pc, entry->fixup); + current->thread.bad_uaddr = address; + regs->pc = entry->fixup; + return; + } + + /* Oops. The kernel tried to access some bad page. We'll have to + * terminate things with extreme prejudice. + */ + pr_alert("Unable to handle kernel paging request at virtual " + "address %08lx\n pc = %08lx, ra = %08lx\n", + address, regs->pc, regs->areg[0]); + die("Oops", regs, sig); + do_exit(sig); +} diff --git a/arch/xtensa/mm/highmem.c b/arch/xtensa/mm/highmem.c new file mode 100644 index 000000000..184ceadcc --- /dev/null +++ b/arch/xtensa/mm/highmem.c @@ -0,0 +1,95 @@ +/* + * High memory support for Xtensa architecture + * + * This file is subject to the terms and conditions of the GNU General + * Public License. See the file "COPYING" in the main directory of + * this archive for more details. + * + * Copyright (C) 2014 Cadence Design Systems Inc. + */ + +#include <linux/export.h> +#include <linux/highmem.h> +#include <asm/tlbflush.h> + +static pte_t *kmap_pte; + +#if DCACHE_WAY_SIZE > PAGE_SIZE +unsigned int last_pkmap_nr_arr[DCACHE_N_COLORS]; +wait_queue_head_t pkmap_map_wait_arr[DCACHE_N_COLORS]; + +static void __init kmap_waitqueues_init(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(pkmap_map_wait_arr); ++i) + init_waitqueue_head(pkmap_map_wait_arr + i); +} +#else +static inline void kmap_waitqueues_init(void) +{ +} +#endif + +static inline enum fixed_addresses kmap_idx(int type, unsigned long color) +{ + return (type + KM_TYPE_NR * smp_processor_id()) * DCACHE_N_COLORS + + color; +} + +void *kmap_atomic(struct page *page) +{ + enum fixed_addresses idx; + unsigned long vaddr; + + preempt_disable(); + pagefault_disable(); + if (!PageHighMem(page)) + return page_address(page); + + idx = kmap_idx(kmap_atomic_idx_push(), + DCACHE_ALIAS(page_to_phys(page))); + vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); +#ifdef CONFIG_DEBUG_HIGHMEM + BUG_ON(!pte_none(*(kmap_pte + idx))); +#endif + set_pte(kmap_pte + idx, mk_pte(page, PAGE_KERNEL_EXEC)); + + return (void *)vaddr; +} +EXPORT_SYMBOL(kmap_atomic); + +void __kunmap_atomic(void *kvaddr) +{ + if (kvaddr >= (void *)FIXADDR_START && + kvaddr < (void *)FIXADDR_TOP) { + int idx = kmap_idx(kmap_atomic_idx(), + DCACHE_ALIAS((unsigned long)kvaddr)); + + /* + * Force other mappings to Oops if they'll try to access this + * pte without first remap it. Keeping stale mappings around + * is a bad idea also, in case the page changes cacheability + * attributes or becomes a protected page in a hypervisor. + */ + pte_clear(&init_mm, kvaddr, kmap_pte + idx); + local_flush_tlb_kernel_range((unsigned long)kvaddr, + (unsigned long)kvaddr + PAGE_SIZE); + + kmap_atomic_idx_pop(); + } + + pagefault_enable(); + preempt_enable(); +} +EXPORT_SYMBOL(__kunmap_atomic); + +void __init kmap_init(void) +{ + unsigned long kmap_vstart; + + /* cache the first kmap pte */ + kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN); + kmap_pte = kmap_get_fixmap_pte(kmap_vstart); + kmap_waitqueues_init(); +} diff --git a/arch/xtensa/mm/init.c b/arch/xtensa/mm/init.c new file mode 100644 index 000000000..34aead7dc --- /dev/null +++ b/arch/xtensa/mm/init.c @@ -0,0 +1,269 @@ +/* + * arch/xtensa/mm/init.c + * + * Derived from MIPS, PPC. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2001 - 2005 Tensilica Inc. + * Copyright (C) 2014 - 2016 Cadence Design Systems Inc. + * + * Chris Zankel <chris@zankel.net> + * Joe Taylor <joe@tensilica.com, joetylr@yahoo.com> + * Marc Gauthier + * Kevin Chea + */ + +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/bootmem.h> +#include <linux/gfp.h> +#include <linux/highmem.h> +#include <linux/swap.h> +#include <linux/mman.h> +#include <linux/nodemask.h> +#include <linux/mm.h> +#include <linux/of_fdt.h> +#include <linux/dma-contiguous.h> + +#include <asm/bootparam.h> +#include <asm/page.h> +#include <asm/sections.h> +#include <asm/sysmem.h> + +/* + * Initialize the bootmem system and give it all low memory we have available. + */ + +void __init bootmem_init(void) +{ + /* Reserve all memory below PHYS_OFFSET, as memory + * accounting doesn't work for pages below that address. + * + * If PHYS_OFFSET is zero reserve page at address 0: + * successfull allocations should never return NULL. + */ + if (PHYS_OFFSET) + memblock_reserve(0, PHYS_OFFSET); + else + memblock_reserve(0, 1); + + early_init_fdt_scan_reserved_mem(); + + if (!memblock_phys_mem_size()) + panic("No memory found!\n"); + + min_low_pfn = PFN_UP(memblock_start_of_DRAM()); + min_low_pfn = max(min_low_pfn, PFN_UP(PHYS_OFFSET)); + max_pfn = PFN_DOWN(memblock_end_of_DRAM()); + max_low_pfn = min(max_pfn, MAX_LOW_PFN); + + memblock_set_current_limit(PFN_PHYS(max_low_pfn)); + dma_contiguous_reserve(PFN_PHYS(max_low_pfn)); + + memblock_dump_all(); +} + + +void __init zones_init(void) +{ + /* All pages are DMA-able, so we put them all in the DMA zone. */ + unsigned long zones_size[MAX_NR_ZONES] = { + [ZONE_DMA] = max_low_pfn - ARCH_PFN_OFFSET, +#ifdef CONFIG_HIGHMEM + [ZONE_HIGHMEM] = max_pfn - max_low_pfn, +#endif + }; + free_area_init_node(0, zones_size, ARCH_PFN_OFFSET, NULL); +} + +#ifdef CONFIG_HIGHMEM +static void __init free_area_high(unsigned long pfn, unsigned long end) +{ + for (; pfn < end; pfn++) + free_highmem_page(pfn_to_page(pfn)); +} + +static void __init free_highpages(void) +{ + unsigned long max_low = max_low_pfn; + struct memblock_region *mem, *res; + + reset_all_zones_managed_pages(); + /* set highmem page free */ + for_each_memblock(memory, mem) { + unsigned long start = memblock_region_memory_base_pfn(mem); + unsigned long end = memblock_region_memory_end_pfn(mem); + + /* Ignore complete lowmem entries */ + if (end <= max_low) + continue; + + if (memblock_is_nomap(mem)) + continue; + + /* Truncate partial highmem entries */ + if (start < max_low) + start = max_low; + + /* Find and exclude any reserved regions */ + for_each_memblock(reserved, res) { + unsigned long res_start, res_end; + + res_start = memblock_region_reserved_base_pfn(res); + res_end = memblock_region_reserved_end_pfn(res); + + if (res_end < start) + continue; + if (res_start < start) + res_start = start; + if (res_start > end) + res_start = end; + if (res_end > end) + res_end = end; + if (res_start != start) + free_area_high(start, res_start); + start = res_end; + if (start == end) + break; + } + + /* And now free anything which remains */ + if (start < end) + free_area_high(start, end); + } +} +#else +static void __init free_highpages(void) +{ +} +#endif + +/* + * Initialize memory pages. + */ + +void __init mem_init(void) +{ + free_highpages(); + + max_mapnr = max_pfn - ARCH_PFN_OFFSET; + high_memory = (void *)__va(max_low_pfn << PAGE_SHIFT); + + free_all_bootmem(); + + mem_init_print_info(NULL); + pr_info("virtual kernel memory layout:\n" +#ifdef CONFIG_KASAN + " kasan : 0x%08lx - 0x%08lx (%5lu MB)\n" +#endif +#ifdef CONFIG_MMU + " vmalloc : 0x%08lx - 0x%08lx (%5lu MB)\n" +#endif +#ifdef CONFIG_HIGHMEM + " pkmap : 0x%08lx - 0x%08lx (%5lu kB)\n" + " fixmap : 0x%08lx - 0x%08lx (%5lu kB)\n" +#endif + " lowmem : 0x%08lx - 0x%08lx (%5lu MB)\n" + " .text : 0x%08lx - 0x%08lx (%5lu kB)\n" + " .rodata : 0x%08lx - 0x%08lx (%5lu kB)\n" + " .data : 0x%08lx - 0x%08lx (%5lu kB)\n" + " .init : 0x%08lx - 0x%08lx (%5lu kB)\n" + " .bss : 0x%08lx - 0x%08lx (%5lu kB)\n", +#ifdef CONFIG_KASAN + KASAN_SHADOW_START, KASAN_SHADOW_START + KASAN_SHADOW_SIZE, + KASAN_SHADOW_SIZE >> 20, +#endif +#ifdef CONFIG_MMU + VMALLOC_START, VMALLOC_END, + (VMALLOC_END - VMALLOC_START) >> 20, +#ifdef CONFIG_HIGHMEM + PKMAP_BASE, PKMAP_BASE + LAST_PKMAP * PAGE_SIZE, + (LAST_PKMAP*PAGE_SIZE) >> 10, + FIXADDR_START, FIXADDR_TOP, + (FIXADDR_TOP - FIXADDR_START) >> 10, +#endif + PAGE_OFFSET, PAGE_OFFSET + + (max_low_pfn - min_low_pfn) * PAGE_SIZE, +#else + min_low_pfn * PAGE_SIZE, max_low_pfn * PAGE_SIZE, +#endif + ((max_low_pfn - min_low_pfn) * PAGE_SIZE) >> 20, + (unsigned long)_text, (unsigned long)_etext, + (unsigned long)(_etext - _text) >> 10, + (unsigned long)__start_rodata, (unsigned long)_sdata, + (unsigned long)(_sdata - __start_rodata) >> 10, + (unsigned long)_sdata, (unsigned long)_edata, + (unsigned long)(_edata - _sdata) >> 10, + (unsigned long)__init_begin, (unsigned long)__init_end, + (unsigned long)(__init_end - __init_begin) >> 10, + (unsigned long)__bss_start, (unsigned long)__bss_stop, + (unsigned long)(__bss_stop - __bss_start) >> 10); +} + +#ifdef CONFIG_BLK_DEV_INITRD +extern int initrd_is_mapped; + +void free_initrd_mem(unsigned long start, unsigned long end) +{ + if (initrd_is_mapped) + free_reserved_area((void *)start, (void *)end, -1, "initrd"); +} +#endif + +void free_initmem(void) +{ + free_initmem_default(-1); +} + +static void __init parse_memmap_one(char *p) +{ + char *oldp; + unsigned long start_at, mem_size; + + if (!p) + return; + + oldp = p; + mem_size = memparse(p, &p); + if (p == oldp) + return; + + switch (*p) { + case '@': + start_at = memparse(p + 1, &p); + memblock_add(start_at, mem_size); + break; + + case '$': + start_at = memparse(p + 1, &p); + memblock_reserve(start_at, mem_size); + break; + + case 0: + memblock_reserve(mem_size, -mem_size); + break; + + default: + pr_warn("Unrecognized memmap syntax: %s\n", p); + break; + } +} + +static int __init parse_memmap_opt(char *str) +{ + while (str) { + char *k = strchr(str, ','); + + if (k) + *k++ = 0; + + parse_memmap_one(str); + str = k; + } + + return 0; +} +early_param("memmap", parse_memmap_opt); diff --git a/arch/xtensa/mm/ioremap.c b/arch/xtensa/mm/ioremap.c new file mode 100644 index 000000000..d89c3c5fd --- /dev/null +++ b/arch/xtensa/mm/ioremap.c @@ -0,0 +1,68 @@ +/* + * ioremap implementation. + * + * Copyright (C) 2015 Cadence Design Systems Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/io.h> +#include <linux/vmalloc.h> +#include <asm/cacheflush.h> +#include <asm/io.h> +#include <asm/pgtable.h> + +static void __iomem *xtensa_ioremap(unsigned long paddr, unsigned long size, + pgprot_t prot) +{ + unsigned long offset = paddr & ~PAGE_MASK; + unsigned long pfn = __phys_to_pfn(paddr); + struct vm_struct *area; + unsigned long vaddr; + int err; + + paddr &= PAGE_MASK; + + WARN_ON(pfn_valid(pfn)); + + size = PAGE_ALIGN(offset + size); + + area = get_vm_area(size, VM_IOREMAP); + if (!area) + return NULL; + + vaddr = (unsigned long)area->addr; + area->phys_addr = paddr; + + err = ioremap_page_range(vaddr, vaddr + size, paddr, prot); + + if (err) { + vunmap((void *)vaddr); + return NULL; + } + + flush_cache_vmap(vaddr, vaddr + size); + return (void __iomem *)(offset + vaddr); +} + +void __iomem *xtensa_ioremap_nocache(unsigned long addr, unsigned long size) +{ + return xtensa_ioremap(addr, size, pgprot_noncached(PAGE_KERNEL)); +} +EXPORT_SYMBOL(xtensa_ioremap_nocache); + +void __iomem *xtensa_ioremap_cache(unsigned long addr, unsigned long size) +{ + return xtensa_ioremap(addr, size, PAGE_KERNEL); +} +EXPORT_SYMBOL(xtensa_ioremap_cache); + +void xtensa_iounmap(volatile void __iomem *io_addr) +{ + void *addr = (void *)(PAGE_MASK & (unsigned long)io_addr); + + vunmap(addr); +} +EXPORT_SYMBOL(xtensa_iounmap); diff --git a/arch/xtensa/mm/kasan_init.c b/arch/xtensa/mm/kasan_init.c new file mode 100644 index 000000000..6b532b6bd --- /dev/null +++ b/arch/xtensa/mm/kasan_init.c @@ -0,0 +1,95 @@ +/* + * Xtensa KASAN shadow map initialization + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2017 Cadence Design Systems Inc. + */ + +#include <linux/bootmem.h> +#include <linux/init_task.h> +#include <linux/kasan.h> +#include <linux/kernel.h> +#include <linux/memblock.h> +#include <asm/initialize_mmu.h> +#include <asm/tlbflush.h> +#include <asm/traps.h> + +void __init kasan_early_init(void) +{ + unsigned long vaddr = KASAN_SHADOW_START; + pgd_t *pgd = pgd_offset_k(vaddr); + pmd_t *pmd = pmd_offset(pgd, vaddr); + int i; + + for (i = 0; i < PTRS_PER_PTE; ++i) + set_pte(kasan_zero_pte + i, + mk_pte(virt_to_page(kasan_zero_page), PAGE_KERNEL)); + + for (vaddr = 0; vaddr < KASAN_SHADOW_SIZE; vaddr += PMD_SIZE, ++pmd) { + BUG_ON(!pmd_none(*pmd)); + set_pmd(pmd, __pmd((unsigned long)kasan_zero_pte)); + } + early_trap_init(); +} + +static void __init populate(void *start, void *end) +{ + unsigned long n_pages = (end - start) / PAGE_SIZE; + unsigned long n_pmds = n_pages / PTRS_PER_PTE; + unsigned long i, j; + unsigned long vaddr = (unsigned long)start; + pgd_t *pgd = pgd_offset_k(vaddr); + pmd_t *pmd = pmd_offset(pgd, vaddr); + pte_t *pte = memblock_virt_alloc(n_pages * sizeof(pte_t), PAGE_SIZE); + + pr_debug("%s: %p - %p\n", __func__, start, end); + + for (i = j = 0; i < n_pmds; ++i) { + int k; + + for (k = 0; k < PTRS_PER_PTE; ++k, ++j) { + phys_addr_t phys = + memblock_alloc_base(PAGE_SIZE, PAGE_SIZE, + MEMBLOCK_ALLOC_ANYWHERE); + + set_pte(pte + j, pfn_pte(PHYS_PFN(phys), PAGE_KERNEL)); + } + } + + for (i = 0; i < n_pmds ; ++i, pte += PTRS_PER_PTE) + set_pmd(pmd + i, __pmd((unsigned long)pte)); + + local_flush_tlb_all(); + memset(start, 0, end - start); +} + +void __init kasan_init(void) +{ + int i; + + BUILD_BUG_ON(KASAN_SHADOW_OFFSET != KASAN_SHADOW_START - + (KASAN_START_VADDR >> KASAN_SHADOW_SCALE_SHIFT)); + BUILD_BUG_ON(VMALLOC_START < KASAN_START_VADDR); + + /* + * Replace shadow map pages that cover addresses from VMALLOC area + * start to the end of KSEG with clean writable pages. + */ + populate(kasan_mem_to_shadow((void *)VMALLOC_START), + kasan_mem_to_shadow((void *)XCHAL_KSEG_BYPASS_VADDR)); + + /* Write protect kasan_zero_page and zero-initialize it again. */ + for (i = 0; i < PTRS_PER_PTE; ++i) + set_pte(kasan_zero_pte + i, + mk_pte(virt_to_page(kasan_zero_page), PAGE_KERNEL_RO)); + + local_flush_tlb_all(); + memset(kasan_zero_page, 0, PAGE_SIZE); + + /* At this point kasan is fully initialized. Enable error messages. */ + current->kasan_depth = 0; + pr_info("KernelAddressSanitizer initialized\n"); +} diff --git a/arch/xtensa/mm/misc.S b/arch/xtensa/mm/misc.S new file mode 100644 index 000000000..11a01c3e9 --- /dev/null +++ b/arch/xtensa/mm/misc.S @@ -0,0 +1,484 @@ +/* + * arch/xtensa/mm/misc.S + * + * Miscellaneous assembly functions. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2001 - 2007 Tensilica Inc. + * + * Chris Zankel <chris@zankel.net> + */ + + +#include <linux/linkage.h> +#include <asm/page.h> +#include <asm/pgtable.h> +#include <asm/asmmacro.h> +#include <asm/cacheasm.h> +#include <asm/tlbflush.h> + + +/* + * clear_page and clear_user_page are the same for non-cache-aliased configs. + * + * clear_page (unsigned long page) + * a2 + */ + +ENTRY(clear_page) + + entry a1, 16 + + movi a3, 0 + __loopi a2, a7, PAGE_SIZE, 32 + s32i a3, a2, 0 + s32i a3, a2, 4 + s32i a3, a2, 8 + s32i a3, a2, 12 + s32i a3, a2, 16 + s32i a3, a2, 20 + s32i a3, a2, 24 + s32i a3, a2, 28 + __endla a2, a7, 32 + + retw + +ENDPROC(clear_page) + +/* + * copy_page and copy_user_page are the same for non-cache-aliased configs. + * + * copy_page (void *to, void *from) + * a2 a3 + */ + +ENTRY(copy_page) + + entry a1, 16 + + __loopi a2, a4, PAGE_SIZE, 32 + + l32i a8, a3, 0 + l32i a9, a3, 4 + s32i a8, a2, 0 + s32i a9, a2, 4 + + l32i a8, a3, 8 + l32i a9, a3, 12 + s32i a8, a2, 8 + s32i a9, a2, 12 + + l32i a8, a3, 16 + l32i a9, a3, 20 + s32i a8, a2, 16 + s32i a9, a2, 20 + + l32i a8, a3, 24 + l32i a9, a3, 28 + s32i a8, a2, 24 + s32i a9, a2, 28 + + addi a2, a2, 32 + addi a3, a3, 32 + + __endl a2, a4 + + retw + +ENDPROC(copy_page) + +#ifdef CONFIG_MMU +/* + * If we have to deal with cache aliasing, we use temporary memory mappings + * to ensure that the source and destination pages have the same color as + * the virtual address. We use way 0 and 1 for temporary mappings in such cases. + * + * The temporary DTLB entries shouldn't be flushed by interrupts, but are + * flushed by preemptive task switches. Special code in the + * fast_second_level_miss handler re-established the temporary mapping. + * It requires that the PPNs for the destination and source addresses are + * in a6, and a7, respectively. + */ + +/* TLB miss exceptions are treated special in the following region */ + +ENTRY(__tlbtemp_mapping_start) + +#if (DCACHE_WAY_SIZE > PAGE_SIZE) + +/* + * clear_page_alias(void *addr, unsigned long paddr) + * a2 a3 + */ + +ENTRY(clear_page_alias) + + entry a1, 32 + + /* Skip setting up a temporary DTLB if not aliased low page. */ + + movi a5, PAGE_OFFSET + movi a6, 0 + beqz a3, 1f + + /* Setup a temporary DTLB for the addr. */ + + addi a6, a3, (PAGE_KERNEL | _PAGE_HW_WRITE) + mov a4, a2 + wdtlb a6, a2 + dsync + +1: movi a3, 0 + __loopi a2, a7, PAGE_SIZE, 32 + s32i a3, a2, 0 + s32i a3, a2, 4 + s32i a3, a2, 8 + s32i a3, a2, 12 + s32i a3, a2, 16 + s32i a3, a2, 20 + s32i a3, a2, 24 + s32i a3, a2, 28 + __endla a2, a7, 32 + + bnez a6, 1f + retw + + /* We need to invalidate the temporary idtlb entry, if any. */ + +1: idtlb a4 + dsync + + retw + +ENDPROC(clear_page_alias) + +/* + * copy_page_alias(void *to, void *from, + * a2 a3 + * unsigned long to_paddr, unsigned long from_paddr) + * a4 a5 + */ + +ENTRY(copy_page_alias) + + entry a1, 32 + + /* Skip setting up a temporary DTLB for destination if not aliased. */ + + movi a6, 0 + movi a7, 0 + beqz a4, 1f + + /* Setup a temporary DTLB for destination. */ + + addi a6, a4, (PAGE_KERNEL | _PAGE_HW_WRITE) + wdtlb a6, a2 + dsync + + /* Skip setting up a temporary DTLB for source if not aliased. */ + +1: beqz a5, 1f + + /* Setup a temporary DTLB for source. */ + + addi a7, a5, PAGE_KERNEL + addi a8, a3, 1 # way1 + + wdtlb a7, a8 + dsync + +1: __loopi a2, a4, PAGE_SIZE, 32 + + l32i a8, a3, 0 + l32i a9, a3, 4 + s32i a8, a2, 0 + s32i a9, a2, 4 + + l32i a8, a3, 8 + l32i a9, a3, 12 + s32i a8, a2, 8 + s32i a9, a2, 12 + + l32i a8, a3, 16 + l32i a9, a3, 20 + s32i a8, a2, 16 + s32i a9, a2, 20 + + l32i a8, a3, 24 + l32i a9, a3, 28 + s32i a8, a2, 24 + s32i a9, a2, 28 + + addi a2, a2, 32 + addi a3, a3, 32 + + __endl a2, a4 + + /* We need to invalidate any temporary mapping! */ + + bnez a6, 1f + bnez a7, 2f + retw + +1: addi a2, a2, -PAGE_SIZE + idtlb a2 + dsync + bnez a7, 2f + retw + +2: addi a3, a3, -PAGE_SIZE+1 + idtlb a3 + dsync + + retw + +ENDPROC(copy_page_alias) + +#endif + +#if (DCACHE_WAY_SIZE > PAGE_SIZE) + +/* + * void __flush_invalidate_dcache_page_alias (addr, phys) + * a2 a3 + */ + +ENTRY(__flush_invalidate_dcache_page_alias) + + entry sp, 16 + + movi a7, 0 # required for exception handler + addi a6, a3, (PAGE_KERNEL | _PAGE_HW_WRITE) + mov a4, a2 + wdtlb a6, a2 + dsync + + ___flush_invalidate_dcache_page a2 a3 + + idtlb a4 + dsync + + retw + +ENDPROC(__flush_invalidate_dcache_page_alias) + +/* + * void __invalidate_dcache_page_alias (addr, phys) + * a2 a3 + */ + +ENTRY(__invalidate_dcache_page_alias) + + entry sp, 16 + + movi a7, 0 # required for exception handler + addi a6, a3, (PAGE_KERNEL | _PAGE_HW_WRITE) + mov a4, a2 + wdtlb a6, a2 + dsync + + ___invalidate_dcache_page a2 a3 + + idtlb a4 + dsync + + retw + +ENDPROC(__invalidate_dcache_page_alias) +#endif + +ENTRY(__tlbtemp_mapping_itlb) + +#if (ICACHE_WAY_SIZE > PAGE_SIZE) + +ENTRY(__invalidate_icache_page_alias) + + entry sp, 16 + + addi a6, a3, (PAGE_KERNEL_EXEC | _PAGE_HW_WRITE) + mov a4, a2 + witlb a6, a2 + isync + + ___invalidate_icache_page a2 a3 + + iitlb a4 + isync + retw + +ENDPROC(__invalidate_icache_page_alias) + +#endif + +/* End of special treatment in tlb miss exception */ + +ENTRY(__tlbtemp_mapping_end) + +#endif /* CONFIG_MMU + +/* + * void __invalidate_icache_page(ulong start) + */ + +ENTRY(__invalidate_icache_page) + + entry sp, 16 + + ___invalidate_icache_page a2 a3 + isync + + retw + +ENDPROC(__invalidate_icache_page) + +/* + * void __invalidate_dcache_page(ulong start) + */ + +ENTRY(__invalidate_dcache_page) + + entry sp, 16 + + ___invalidate_dcache_page a2 a3 + dsync + + retw + +ENDPROC(__invalidate_dcache_page) + +/* + * void __flush_invalidate_dcache_page(ulong start) + */ + +ENTRY(__flush_invalidate_dcache_page) + + entry sp, 16 + + ___flush_invalidate_dcache_page a2 a3 + + dsync + retw + +ENDPROC(__flush_invalidate_dcache_page) + +/* + * void __flush_dcache_page(ulong start) + */ + +ENTRY(__flush_dcache_page) + + entry sp, 16 + + ___flush_dcache_page a2 a3 + + dsync + retw + +ENDPROC(__flush_dcache_page) + +/* + * void __invalidate_icache_range(ulong start, ulong size) + */ + +ENTRY(__invalidate_icache_range) + + entry sp, 16 + + ___invalidate_icache_range a2 a3 a4 + isync + + retw + +ENDPROC(__invalidate_icache_range) + +/* + * void __flush_invalidate_dcache_range(ulong start, ulong size) + */ + +ENTRY(__flush_invalidate_dcache_range) + + entry sp, 16 + + ___flush_invalidate_dcache_range a2 a3 a4 + dsync + + retw + +ENDPROC(__flush_invalidate_dcache_range) + +/* + * void _flush_dcache_range(ulong start, ulong size) + */ + +ENTRY(__flush_dcache_range) + + entry sp, 16 + + ___flush_dcache_range a2 a3 a4 + dsync + + retw + +ENDPROC(__flush_dcache_range) + +/* + * void _invalidate_dcache_range(ulong start, ulong size) + */ + +ENTRY(__invalidate_dcache_range) + + entry sp, 16 + + ___invalidate_dcache_range a2 a3 a4 + + retw + +ENDPROC(__invalidate_dcache_range) + +/* + * void _invalidate_icache_all(void) + */ + +ENTRY(__invalidate_icache_all) + + entry sp, 16 + + ___invalidate_icache_all a2 a3 + isync + + retw + +ENDPROC(__invalidate_icache_all) + +/* + * void _flush_invalidate_dcache_all(void) + */ + +ENTRY(__flush_invalidate_dcache_all) + + entry sp, 16 + + ___flush_invalidate_dcache_all a2 a3 + dsync + + retw + +ENDPROC(__flush_invalidate_dcache_all) + +/* + * void _invalidate_dcache_all(void) + */ + +ENTRY(__invalidate_dcache_all) + + entry sp, 16 + + ___invalidate_dcache_all a2 a3 + dsync + + retw + +ENDPROC(__invalidate_dcache_all) diff --git a/arch/xtensa/mm/mmu.c b/arch/xtensa/mm/mmu.c new file mode 100644 index 000000000..470843188 --- /dev/null +++ b/arch/xtensa/mm/mmu.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * xtensa mmu stuff + * + * Extracted from init.c + */ +#include <linux/bootmem.h> +#include <linux/percpu.h> +#include <linux/init.h> +#include <linux/string.h> +#include <linux/slab.h> +#include <linux/cache.h> + +#include <asm/tlb.h> +#include <asm/tlbflush.h> +#include <asm/mmu_context.h> +#include <asm/page.h> +#include <asm/initialize_mmu.h> +#include <asm/io.h> + +#if defined(CONFIG_HIGHMEM) +static void * __init init_pmd(unsigned long vaddr, unsigned long n_pages) +{ + pgd_t *pgd = pgd_offset_k(vaddr); + pmd_t *pmd = pmd_offset(pgd, vaddr); + pte_t *pte; + unsigned long i; + + n_pages = ALIGN(n_pages, PTRS_PER_PTE); + + pr_debug("%s: vaddr: 0x%08lx, n_pages: %ld\n", + __func__, vaddr, n_pages); + + pte = alloc_bootmem_low_pages(n_pages * sizeof(pte_t)); + + for (i = 0; i < n_pages; ++i) + pte_clear(NULL, 0, pte + i); + + for (i = 0; i < n_pages; i += PTRS_PER_PTE, ++pmd) { + pte_t *cur_pte = pte + i; + + BUG_ON(!pmd_none(*pmd)); + set_pmd(pmd, __pmd(((unsigned long)cur_pte) & PAGE_MASK)); + BUG_ON(cur_pte != pte_offset_kernel(pmd, 0)); + pr_debug("%s: pmd: 0x%p, pte: 0x%p\n", + __func__, pmd, cur_pte); + } + return pte; +} + +static void __init fixedrange_init(void) +{ + init_pmd(__fix_to_virt(0), __end_of_fixed_addresses); +} +#endif + +void __init paging_init(void) +{ +#ifdef CONFIG_HIGHMEM + fixedrange_init(); + pkmap_page_table = init_pmd(PKMAP_BASE, LAST_PKMAP); + kmap_init(); +#endif +} + +/* + * Flush the mmu and reset associated register to default values. + */ +void init_mmu(void) +{ +#if !(XCHAL_HAVE_PTP_MMU && XCHAL_HAVE_SPANNING_WAY) + /* + * Writing zeros to the instruction and data TLBCFG special + * registers ensure that valid values exist in the register. + * + * For existing PGSZID<w> fields, zero selects the first element + * of the page-size array. For nonexistent PGSZID<w> fields, + * zero is the best value to write. Also, when changing PGSZID<w> + * fields, the corresponding TLB must be flushed. + */ + set_itlbcfg_register(0); + set_dtlbcfg_register(0); +#endif + init_kio(); + local_flush_tlb_all(); + + /* Set rasid register to a known value. */ + + set_rasid_register(ASID_INSERT(ASID_USER_FIRST)); + + /* Set PTEVADDR special register to the start of the page + * table, which is in kernel mappable space (ie. not + * statically mapped). This register's value is undefined on + * reset. + */ + set_ptevaddr_register(XCHAL_PAGE_TABLE_VADDR); +} + +void init_kio(void) +{ +#if XCHAL_HAVE_PTP_MMU && XCHAL_HAVE_SPANNING_WAY && defined(CONFIG_USE_OF) + /* + * Update the IO area mapping in case xtensa_kio_paddr has changed + */ + write_dtlb_entry(__pte(xtensa_kio_paddr + CA_WRITEBACK), + XCHAL_KIO_CACHED_VADDR + 6); + write_itlb_entry(__pte(xtensa_kio_paddr + CA_WRITEBACK), + XCHAL_KIO_CACHED_VADDR + 6); + write_dtlb_entry(__pte(xtensa_kio_paddr + CA_BYPASS), + XCHAL_KIO_BYPASS_VADDR + 6); + write_itlb_entry(__pte(xtensa_kio_paddr + CA_BYPASS), + XCHAL_KIO_BYPASS_VADDR + 6); +#endif +} diff --git a/arch/xtensa/mm/tlb.c b/arch/xtensa/mm/tlb.c new file mode 100644 index 000000000..b43f03620 --- /dev/null +++ b/arch/xtensa/mm/tlb.c @@ -0,0 +1,276 @@ +/* + * arch/xtensa/mm/tlb.c + * + * Logic that manipulates the Xtensa MMU. Derived from MIPS. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2001 - 2003 Tensilica Inc. + * + * Joe Taylor + * Chris Zankel <chris@zankel.net> + * Marc Gauthier + */ + +#include <linux/mm.h> +#include <asm/processor.h> +#include <asm/mmu_context.h> +#include <asm/tlbflush.h> +#include <asm/cacheflush.h> + + +static inline void __flush_itlb_all (void) +{ + int w, i; + + for (w = 0; w < ITLB_ARF_WAYS; w++) { + for (i = 0; i < (1 << XCHAL_ITLB_ARF_ENTRIES_LOG2); i++) { + int e = w + (i << PAGE_SHIFT); + invalidate_itlb_entry_no_isync(e); + } + } + asm volatile ("isync\n"); +} + +static inline void __flush_dtlb_all (void) +{ + int w, i; + + for (w = 0; w < DTLB_ARF_WAYS; w++) { + for (i = 0; i < (1 << XCHAL_DTLB_ARF_ENTRIES_LOG2); i++) { + int e = w + (i << PAGE_SHIFT); + invalidate_dtlb_entry_no_isync(e); + } + } + asm volatile ("isync\n"); +} + + +void local_flush_tlb_all(void) +{ + __flush_itlb_all(); + __flush_dtlb_all(); +} + +/* If mm is current, we simply assign the current task a new ASID, thus, + * invalidating all previous tlb entries. If mm is someone else's user mapping, + * wie invalidate the context, thus, when that user mapping is swapped in, + * a new context will be assigned to it. + */ + +void local_flush_tlb_mm(struct mm_struct *mm) +{ + int cpu = smp_processor_id(); + + if (mm == current->active_mm) { + unsigned long flags; + local_irq_save(flags); + mm->context.asid[cpu] = NO_CONTEXT; + activate_context(mm, cpu); + local_irq_restore(flags); + } else { + mm->context.asid[cpu] = NO_CONTEXT; + mm->context.cpu = -1; + } +} + + +#define _ITLB_ENTRIES (ITLB_ARF_WAYS << XCHAL_ITLB_ARF_ENTRIES_LOG2) +#define _DTLB_ENTRIES (DTLB_ARF_WAYS << XCHAL_DTLB_ARF_ENTRIES_LOG2) +#if _ITLB_ENTRIES > _DTLB_ENTRIES +# define _TLB_ENTRIES _ITLB_ENTRIES +#else +# define _TLB_ENTRIES _DTLB_ENTRIES +#endif + +void local_flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + int cpu = smp_processor_id(); + struct mm_struct *mm = vma->vm_mm; + unsigned long flags; + + if (mm->context.asid[cpu] == NO_CONTEXT) + return; + + pr_debug("[tlbrange<%02lx,%08lx,%08lx>]\n", + (unsigned long)mm->context.asid[cpu], start, end); + local_irq_save(flags); + + if (end-start + (PAGE_SIZE-1) <= _TLB_ENTRIES << PAGE_SHIFT) { + int oldpid = get_rasid_register(); + + set_rasid_register(ASID_INSERT(mm->context.asid[cpu])); + start &= PAGE_MASK; + if (vma->vm_flags & VM_EXEC) + while(start < end) { + invalidate_itlb_mapping(start); + invalidate_dtlb_mapping(start); + start += PAGE_SIZE; + } + else + while(start < end) { + invalidate_dtlb_mapping(start); + start += PAGE_SIZE; + } + + set_rasid_register(oldpid); + } else { + local_flush_tlb_mm(mm); + } + local_irq_restore(flags); +} + +void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + int cpu = smp_processor_id(); + struct mm_struct* mm = vma->vm_mm; + unsigned long flags; + int oldpid; + + if (mm->context.asid[cpu] == NO_CONTEXT) + return; + + local_irq_save(flags); + + oldpid = get_rasid_register(); + set_rasid_register(ASID_INSERT(mm->context.asid[cpu])); + + if (vma->vm_flags & VM_EXEC) + invalidate_itlb_mapping(page); + invalidate_dtlb_mapping(page); + + set_rasid_register(oldpid); + + local_irq_restore(flags); +} + +void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + if (end > start && start >= TASK_SIZE && end <= PAGE_OFFSET && + end - start < _TLB_ENTRIES << PAGE_SHIFT) { + start &= PAGE_MASK; + while (start < end) { + invalidate_itlb_mapping(start); + invalidate_dtlb_mapping(start); + start += PAGE_SIZE; + } + } else { + local_flush_tlb_all(); + } +} + +#ifdef CONFIG_DEBUG_TLB_SANITY + +static unsigned get_pte_for_vaddr(unsigned vaddr) +{ + struct task_struct *task = get_current(); + struct mm_struct *mm = task->mm; + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + + if (!mm) + mm = task->active_mm; + pgd = pgd_offset(mm, vaddr); + if (pgd_none_or_clear_bad(pgd)) + return 0; + pmd = pmd_offset(pgd, vaddr); + if (pmd_none_or_clear_bad(pmd)) + return 0; + pte = pte_offset_map(pmd, vaddr); + if (!pte) + return 0; + return pte_val(*pte); +} + +enum { + TLB_SUSPICIOUS = 1, + TLB_INSANE = 2, +}; + +static void tlb_insane(void) +{ + BUG_ON(1); +} + +static void tlb_suspicious(void) +{ + WARN_ON(1); +} + +/* + * Check that TLB entries with kernel ASID (1) have kernel VMA (>= TASK_SIZE), + * and TLB entries with user ASID (>=4) have VMA < TASK_SIZE. + * + * Check that valid TLB entries either have the same PA as the PTE, or PTE is + * marked as non-present. Non-present PTE and the page with non-zero refcount + * and zero mapcount is normal for batched TLB flush operation. Zero refcount + * means that the page was freed prematurely. Non-zero mapcount is unusual, + * but does not necessary means an error, thus marked as suspicious. + */ +static int check_tlb_entry(unsigned w, unsigned e, bool dtlb) +{ + unsigned tlbidx = w | (e << PAGE_SHIFT); + unsigned r0 = dtlb ? + read_dtlb_virtual(tlbidx) : read_itlb_virtual(tlbidx); + unsigned r1 = dtlb ? + read_dtlb_translation(tlbidx) : read_itlb_translation(tlbidx); + unsigned vpn = (r0 & PAGE_MASK) | (e << PAGE_SHIFT); + unsigned pte = get_pte_for_vaddr(vpn); + unsigned mm_asid = (get_rasid_register() >> 8) & ASID_MASK; + unsigned tlb_asid = r0 & ASID_MASK; + bool kernel = tlb_asid == 1; + int rc = 0; + + if (tlb_asid > 0 && ((vpn < TASK_SIZE) == kernel)) { + pr_err("%cTLB: way: %u, entry: %u, VPN %08x in %s PTE\n", + dtlb ? 'D' : 'I', w, e, vpn, + kernel ? "kernel" : "user"); + rc |= TLB_INSANE; + } + + if (tlb_asid == mm_asid) { + if ((pte ^ r1) & PAGE_MASK) { + pr_err("%cTLB: way: %u, entry: %u, mapping: %08x->%08x, PTE: %08x\n", + dtlb ? 'D' : 'I', w, e, r0, r1, pte); + if (pte == 0 || !pte_present(__pte(pte))) { + struct page *p = pfn_to_page(r1 >> PAGE_SHIFT); + pr_err("page refcount: %d, mapcount: %d\n", + page_count(p), + page_mapcount(p)); + if (!page_count(p)) + rc |= TLB_INSANE; + else if (page_mapcount(p)) + rc |= TLB_SUSPICIOUS; + } else { + rc |= TLB_INSANE; + } + } + } + return rc; +} + +void check_tlb_sanity(void) +{ + unsigned long flags; + unsigned w, e; + int bug = 0; + + local_irq_save(flags); + for (w = 0; w < DTLB_ARF_WAYS; ++w) + for (e = 0; e < (1 << XCHAL_DTLB_ARF_ENTRIES_LOG2); ++e) + bug |= check_tlb_entry(w, e, true); + for (w = 0; w < ITLB_ARF_WAYS; ++w) + for (e = 0; e < (1 << XCHAL_ITLB_ARF_ENTRIES_LOG2); ++e) + bug |= check_tlb_entry(w, e, false); + if (bug & TLB_INSANE) + tlb_insane(); + if (bug & TLB_SUSPICIOUS) + tlb_suspicious(); + local_irq_restore(flags); +} + +#endif /* CONFIG_DEBUG_TLB_SANITY */ |