diff options
Diffstat (limited to 'arch/mips/kernel/genex.S')
| -rw-r--r-- | arch/mips/kernel/genex.S | 682 |
1 files changed, 682 insertions, 0 deletions
diff --git a/arch/mips/kernel/genex.S b/arch/mips/kernel/genex.S new file mode 100644 index 000000000..bcce32a3d --- /dev/null +++ b/arch/mips/kernel/genex.S @@ -0,0 +1,682 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1994 - 2000, 2001, 2003 Ralf Baechle + * Copyright (C) 1999, 2000 Silicon Graphics, Inc. + * Copyright (C) 2002, 2007 Maciej W. Rozycki + * Copyright (C) 2001, 2012 MIPS Technologies, Inc. All rights reserved. + */ +#include <linux/init.h> + +#include <asm/asm.h> +#include <asm/asmmacro.h> +#include <asm/cacheops.h> +#include <asm/irqflags.h> +#include <asm/regdef.h> +#include <asm/fpregdef.h> +#include <asm/mipsregs.h> +#include <asm/stackframe.h> +#include <asm/sync.h> +#include <asm/war.h> +#include <asm/thread_info.h> + + __INIT + +/* + * General exception vector for all other CPUs. + * + * Be careful when changing this, it has to be at most 128 bytes + * to fit into space reserved for the exception handler. + */ +NESTED(except_vec3_generic, 0, sp) + .set push + .set noat + mfc0 k1, CP0_CAUSE + andi k1, k1, 0x7c +#ifdef CONFIG_64BIT + dsll k1, k1, 1 +#endif + PTR_L k0, exception_handlers(k1) + jr k0 + .set pop + END(except_vec3_generic) + +/* + * General exception handler for CPUs with virtual coherency exception. + * + * Be careful when changing this, it has to be at most 256 (as a special + * exception) bytes to fit into space reserved for the exception handler. + */ +NESTED(except_vec3_r4000, 0, sp) + .set push + .set arch=r4000 + .set noat + mfc0 k1, CP0_CAUSE + li k0, 31<<2 + andi k1, k1, 0x7c + .set push + .set noreorder + .set nomacro + beq k1, k0, handle_vced + li k0, 14<<2 + beq k1, k0, handle_vcei +#ifdef CONFIG_64BIT + dsll k1, k1, 1 +#endif + .set pop + PTR_L k0, exception_handlers(k1) + jr k0 + + /* + * Big shit, we now may have two dirty primary cache lines for the same + * physical address. We can safely invalidate the line pointed to by + * c0_badvaddr because after return from this exception handler the + * load / store will be re-executed. + */ +handle_vced: + MFC0 k0, CP0_BADVADDR + li k1, -4 # Is this ... + and k0, k1 # ... really needed? + mtc0 zero, CP0_TAGLO + cache Index_Store_Tag_D, (k0) + cache Hit_Writeback_Inv_SD, (k0) +#ifdef CONFIG_PROC_FS + PTR_LA k0, vced_count + lw k1, (k0) + addiu k1, 1 + sw k1, (k0) +#endif + eret + +handle_vcei: + MFC0 k0, CP0_BADVADDR + cache Hit_Writeback_Inv_SD, (k0) # also cleans pi +#ifdef CONFIG_PROC_FS + PTR_LA k0, vcei_count + lw k1, (k0) + addiu k1, 1 + sw k1, (k0) +#endif + eret + .set pop + END(except_vec3_r4000) + + __FINIT + + .align 5 /* 32 byte rollback region */ +LEAF(__r4k_wait) + .set push + .set noreorder + /* start of rollback region */ + LONG_L t0, TI_FLAGS($28) + nop + andi t0, _TIF_NEED_RESCHED + bnez t0, 1f + nop + nop + nop +#ifdef CONFIG_CPU_MICROMIPS + nop + nop + nop + nop +#endif + .set MIPS_ISA_ARCH_LEVEL_RAW + wait + /* end of rollback region (the region size must be power of two) */ +1: + jr ra + nop + .set pop + END(__r4k_wait) + + .macro BUILD_ROLLBACK_PROLOGUE handler + FEXPORT(rollback_\handler) + .set push + .set noat + MFC0 k0, CP0_EPC + PTR_LA k1, __r4k_wait + ori k0, 0x1f /* 32 byte rollback region */ + xori k0, 0x1f + bne k0, k1, \handler + MTC0 k0, CP0_EPC + .set pop + .endm + + .align 5 +BUILD_ROLLBACK_PROLOGUE handle_int +NESTED(handle_int, PT_SIZE, sp) + .cfi_signal_frame +#ifdef CONFIG_TRACE_IRQFLAGS + /* + * Check to see if the interrupted code has just disabled + * interrupts and ignore this interrupt for now if so. + * + * local_irq_disable() disables interrupts and then calls + * trace_hardirqs_off() to track the state. If an interrupt is taken + * after interrupts are disabled but before the state is updated + * it will appear to restore_all that it is incorrectly returning with + * interrupts disabled + */ + .set push + .set noat + mfc0 k0, CP0_STATUS +#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX) + and k0, ST0_IEP + bnez k0, 1f + + mfc0 k0, CP0_EPC + .set noreorder + j k0 + rfe +#else + and k0, ST0_IE + bnez k0, 1f + + eret +#endif +1: + .set pop +#endif + SAVE_ALL docfi=1 + CLI + TRACE_IRQS_OFF + + LONG_L s0, TI_REGS($28) + LONG_S sp, TI_REGS($28) + + /* + * SAVE_ALL ensures we are using a valid kernel stack for the thread. + * Check if we are already using the IRQ stack. + */ + move s1, sp # Preserve the sp + + /* Get IRQ stack for this CPU */ + ASM_CPUID_MFC0 k0, ASM_SMP_CPUID_REG +#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32) + lui k1, %hi(irq_stack) +#else + lui k1, %highest(irq_stack) + daddiu k1, %higher(irq_stack) + dsll k1, 16 + daddiu k1, %hi(irq_stack) + dsll k1, 16 +#endif + LONG_SRL k0, SMP_CPUID_PTRSHIFT + LONG_ADDU k1, k0 + LONG_L t0, %lo(irq_stack)(k1) + + # Check if already on IRQ stack + PTR_LI t1, ~(_THREAD_SIZE-1) + and t1, t1, sp + beq t0, t1, 2f + + /* Switch to IRQ stack */ + li t1, _IRQ_STACK_START + PTR_ADD sp, t0, t1 + + /* Save task's sp on IRQ stack so that unwinding can follow it */ + LONG_S s1, 0(sp) +2: + jal plat_irq_dispatch + + /* Restore sp */ + move sp, s1 + + j ret_from_irq +#ifdef CONFIG_CPU_MICROMIPS + nop +#endif + END(handle_int) + + __INIT + +/* + * Special interrupt vector for MIPS64 ISA & embedded MIPS processors. + * This is a dedicated interrupt exception vector which reduces the + * interrupt processing overhead. The jump instruction will be replaced + * at the initialization time. + * + * Be careful when changing this, it has to be at most 128 bytes + * to fit into space reserved for the exception handler. + */ +NESTED(except_vec4, 0, sp) +1: j 1b /* Dummy, will be replaced */ + END(except_vec4) + +/* + * EJTAG debug exception handler. + * The EJTAG debug exception entry point is 0xbfc00480, which + * normally is in the boot PROM, so the boot PROM must do an + * unconditional jump to this vector. + */ +NESTED(except_vec_ejtag_debug, 0, sp) + j ejtag_debug_handler +#ifdef CONFIG_CPU_MICROMIPS + nop +#endif + END(except_vec_ejtag_debug) + + __FINIT + +/* + * Vectored interrupt handler. + * This prototype is copied to ebase + n*IntCtl.VS and patched + * to invoke the handler + */ +BUILD_ROLLBACK_PROLOGUE except_vec_vi +NESTED(except_vec_vi, 0, sp) + SAVE_SOME docfi=1 + SAVE_AT docfi=1 + .set push + .set noreorder + PTR_LA v1, except_vec_vi_handler +FEXPORT(except_vec_vi_lui) + lui v0, 0 /* Patched */ + jr v1 +FEXPORT(except_vec_vi_ori) + ori v0, 0 /* Patched */ + .set pop + END(except_vec_vi) +EXPORT(except_vec_vi_end) + +/* + * Common Vectored Interrupt code + * Complete the register saves and invoke the handler which is passed in $v0 + */ +NESTED(except_vec_vi_handler, 0, sp) + SAVE_TEMP + SAVE_STATIC + CLI +#ifdef CONFIG_TRACE_IRQFLAGS + move s0, v0 + TRACE_IRQS_OFF + move v0, s0 +#endif + + LONG_L s0, TI_REGS($28) + LONG_S sp, TI_REGS($28) + + /* + * SAVE_ALL ensures we are using a valid kernel stack for the thread. + * Check if we are already using the IRQ stack. + */ + move s1, sp # Preserve the sp + + /* Get IRQ stack for this CPU */ + ASM_CPUID_MFC0 k0, ASM_SMP_CPUID_REG +#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32) + lui k1, %hi(irq_stack) +#else + lui k1, %highest(irq_stack) + daddiu k1, %higher(irq_stack) + dsll k1, 16 + daddiu k1, %hi(irq_stack) + dsll k1, 16 +#endif + LONG_SRL k0, SMP_CPUID_PTRSHIFT + LONG_ADDU k1, k0 + LONG_L t0, %lo(irq_stack)(k1) + + # Check if already on IRQ stack + PTR_LI t1, ~(_THREAD_SIZE-1) + and t1, t1, sp + beq t0, t1, 2f + + /* Switch to IRQ stack */ + li t1, _IRQ_STACK_START + PTR_ADD sp, t0, t1 + + /* Save task's sp on IRQ stack so that unwinding can follow it */ + LONG_S s1, 0(sp) +2: + jalr v0 + + /* Restore sp */ + move sp, s1 + + j ret_from_irq + END(except_vec_vi_handler) + +/* + * EJTAG debug exception handler. + */ +NESTED(ejtag_debug_handler, PT_SIZE, sp) + .set push + .set noat + MTC0 k0, CP0_DESAVE + mfc0 k0, CP0_DEBUG + + sll k0, k0, 30 # Check for SDBBP. + bgez k0, ejtag_return + +#ifdef CONFIG_SMP +1: PTR_LA k0, ejtag_debug_buffer_spinlock + __SYNC(full, loongson3_war) +2: ll k0, 0(k0) + bnez k0, 2b + PTR_LA k0, ejtag_debug_buffer_spinlock + sc k0, 0(k0) + beqz k0, 1b +# ifdef CONFIG_WEAK_REORDERING_BEYOND_LLSC + sync +# endif + + PTR_LA k0, ejtag_debug_buffer + LONG_S k1, 0(k0) + + ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG + PTR_SRL k1, SMP_CPUID_PTRSHIFT + PTR_SLL k1, LONGLOG + PTR_LA k0, ejtag_debug_buffer_per_cpu + PTR_ADDU k0, k1 + + PTR_LA k1, ejtag_debug_buffer + LONG_L k1, 0(k1) + LONG_S k1, 0(k0) + + PTR_LA k0, ejtag_debug_buffer_spinlock + sw zero, 0(k0) +#else + PTR_LA k0, ejtag_debug_buffer + LONG_S k1, 0(k0) +#endif + + SAVE_ALL + move a0, sp + jal ejtag_exception_handler + RESTORE_ALL + +#ifdef CONFIG_SMP + ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG + PTR_SRL k1, SMP_CPUID_PTRSHIFT + PTR_SLL k1, LONGLOG + PTR_LA k0, ejtag_debug_buffer_per_cpu + PTR_ADDU k0, k1 + LONG_L k1, 0(k0) +#else + PTR_LA k0, ejtag_debug_buffer + LONG_L k1, 0(k0) +#endif + +ejtag_return: + back_to_back_c0_hazard + MFC0 k0, CP0_DESAVE + .set mips32 + deret + .set pop + END(ejtag_debug_handler) + +/* + * This buffer is reserved for the use of the EJTAG debug + * handler. + */ + .data +EXPORT(ejtag_debug_buffer) + .fill LONGSIZE +#ifdef CONFIG_SMP +EXPORT(ejtag_debug_buffer_spinlock) + .fill LONGSIZE +EXPORT(ejtag_debug_buffer_per_cpu) + .fill LONGSIZE * NR_CPUS +#endif + .previous + + __INIT + +/* + * NMI debug exception handler for MIPS reference boards. + * The NMI debug exception entry point is 0xbfc00000, which + * normally is in the boot PROM, so the boot PROM must do a + * unconditional jump to this vector. + */ +NESTED(except_vec_nmi, 0, sp) + j nmi_handler +#ifdef CONFIG_CPU_MICROMIPS + nop +#endif + END(except_vec_nmi) + + __FINIT + +NESTED(nmi_handler, PT_SIZE, sp) + .cfi_signal_frame + .set push + .set noat + /* + * Clear ERL - restore segment mapping + * Clear BEV - required for page fault exception handler to work + */ + mfc0 k0, CP0_STATUS + ori k0, k0, ST0_EXL + li k1, ~(ST0_BEV | ST0_ERL) + and k0, k0, k1 + mtc0 k0, CP0_STATUS + _ehb + SAVE_ALL + move a0, sp + jal nmi_exception_handler + /* nmi_exception_handler never returns */ + .set pop + END(nmi_handler) + + .macro __build_clear_none + .endm + + .macro __build_clear_sti + TRACE_IRQS_ON + STI + .endm + + .macro __build_clear_cli + CLI + TRACE_IRQS_OFF + .endm + + .macro __build_clear_fpe + CLI + TRACE_IRQS_OFF + .set push + /* gas fails to assemble cfc1 for some archs (octeon).*/ \ + .set mips1 + SET_HARDFLOAT + cfc1 a1, fcr31 + .set pop + .endm + + .macro __build_clear_msa_fpe + CLI + TRACE_IRQS_OFF + _cfcmsa a1, MSA_CSR + .endm + + .macro __build_clear_ade + MFC0 t0, CP0_BADVADDR + PTR_S t0, PT_BVADDR(sp) + KMODE + .endm + + .macro __build_clear_gsexc + .set push + /* + * We need to specify a selector to access the CP0.Diag1 (GSCause) + * register. All GSExc-equipped processors have MIPS32. + */ + .set mips32 + mfc0 a1, CP0_DIAGNOSTIC1 + .set pop + TRACE_IRQS_ON + STI + .endm + + .macro __BUILD_silent exception + .endm + + /* Gas tries to parse the ASM_PRINT argument as a string containing + string escapes and emits bogus warnings if it believes to + recognize an unknown escape code. So make the arguments + start with an n and gas will believe \n is ok ... */ + .macro __BUILD_verbose nexception + LONG_L a1, PT_EPC(sp) +#ifdef CONFIG_32BIT + ASM_PRINT("Got \nexception at %08lx\012") +#endif +#ifdef CONFIG_64BIT + ASM_PRINT("Got \nexception at %016lx\012") +#endif + .endm + + .macro __BUILD_count exception + LONG_L t0,exception_count_\exception + LONG_ADDIU t0, 1 + LONG_S t0,exception_count_\exception + .comm exception_count\exception, 8, 8 + .endm + + .macro __BUILD_HANDLER exception handler clear verbose ext + .align 5 + NESTED(handle_\exception, PT_SIZE, sp) + .cfi_signal_frame + .set noat + SAVE_ALL + FEXPORT(handle_\exception\ext) + __build_clear_\clear + .set at + __BUILD_\verbose \exception + move a0, sp + jal do_\handler + j ret_from_exception + END(handle_\exception) + .endm + + .macro BUILD_HANDLER exception handler clear verbose + __BUILD_HANDLER \exception \handler \clear \verbose _int + .endm + + BUILD_HANDLER adel ade ade silent /* #4 */ + BUILD_HANDLER ades ade ade silent /* #5 */ + BUILD_HANDLER ibe be cli silent /* #6 */ + BUILD_HANDLER dbe be cli silent /* #7 */ + BUILD_HANDLER bp bp sti silent /* #9 */ + BUILD_HANDLER ri ri sti silent /* #10 */ + BUILD_HANDLER cpu cpu sti silent /* #11 */ + BUILD_HANDLER ov ov sti silent /* #12 */ + BUILD_HANDLER tr tr sti silent /* #13 */ + BUILD_HANDLER msa_fpe msa_fpe msa_fpe silent /* #14 */ +#ifdef CONFIG_MIPS_FP_SUPPORT + BUILD_HANDLER fpe fpe fpe silent /* #15 */ +#endif + BUILD_HANDLER ftlb ftlb none silent /* #16 */ + BUILD_HANDLER gsexc gsexc gsexc silent /* #16 */ + BUILD_HANDLER msa msa sti silent /* #21 */ + BUILD_HANDLER mdmx mdmx sti silent /* #22 */ +#ifdef CONFIG_HARDWARE_WATCHPOINTS + /* + * For watch, interrupts will be enabled after the watch + * registers are read. + */ + BUILD_HANDLER watch watch cli silent /* #23 */ +#else + BUILD_HANDLER watch watch sti verbose /* #23 */ +#endif + BUILD_HANDLER mcheck mcheck cli verbose /* #24 */ + BUILD_HANDLER mt mt sti silent /* #25 */ + BUILD_HANDLER dsp dsp sti silent /* #26 */ + BUILD_HANDLER reserved reserved sti verbose /* others */ + + .align 5 + LEAF(handle_ri_rdhwr_tlbp) + .set push + .set noat + .set noreorder + /* check if TLB contains a entry for EPC */ + MFC0 k1, CP0_ENTRYHI + andi k1, MIPS_ENTRYHI_ASID | MIPS_ENTRYHI_ASIDX + MFC0 k0, CP0_EPC + PTR_SRL k0, _PAGE_SHIFT + 1 + PTR_SLL k0, _PAGE_SHIFT + 1 + or k1, k0 + MTC0 k1, CP0_ENTRYHI + mtc0_tlbw_hazard + tlbp + tlb_probe_hazard + mfc0 k1, CP0_INDEX + .set pop + bltz k1, handle_ri /* slow path */ + /* fall thru */ + END(handle_ri_rdhwr_tlbp) + + LEAF(handle_ri_rdhwr) + .set push + .set noat + .set noreorder + /* MIPS32: 0x7c03e83b: rdhwr v1,$29 */ + /* microMIPS: 0x007d6b3c: rdhwr v1,$29 */ + MFC0 k1, CP0_EPC +#if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_MIPS32_R2) || defined(CONFIG_CPU_MIPS64_R2) + and k0, k1, 1 + beqz k0, 1f + xor k1, k0 + lhu k0, (k1) + lhu k1, 2(k1) + ins k1, k0, 16, 16 + lui k0, 0x007d + b docheck + ori k0, 0x6b3c +1: + lui k0, 0x7c03 + lw k1, (k1) + ori k0, 0xe83b +#else + andi k0, k1, 1 + bnez k0, handle_ri + lui k0, 0x7c03 + lw k1, (k1) + ori k0, 0xe83b +#endif + .set reorder +docheck: + bne k0, k1, handle_ri /* if not ours */ + +isrdhwr: + /* The insn is rdhwr. No need to check CAUSE.BD here. */ + get_saved_sp /* k1 := current_thread_info */ + .set noreorder + MFC0 k0, CP0_EPC +#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX) + ori k1, _THREAD_MASK + xori k1, _THREAD_MASK + LONG_L v1, TI_TP_VALUE(k1) + LONG_ADDIU k0, 4 + jr k0 + rfe +#else +#ifndef CONFIG_CPU_DADDI_WORKAROUNDS + LONG_ADDIU k0, 4 /* stall on $k0 */ +#else + .set at=v1 + LONG_ADDIU k0, 4 + .set noat +#endif + MTC0 k0, CP0_EPC + /* I hope three instructions between MTC0 and ERET are enough... */ + ori k1, _THREAD_MASK + xori k1, _THREAD_MASK + LONG_L v1, TI_TP_VALUE(k1) + .set push + .set arch=r4000 + eret + .set pop +#endif + .set pop + END(handle_ri_rdhwr) + +#ifdef CONFIG_CPU_R4X00_BUGS64 +/* A temporary overflow handler used by check_daddi(). */ + + __INIT + + BUILD_HANDLER daddi_ov daddi_ov none silent /* #12 */ +#endif |