Adding upstream version 5.10.209.upstream/5.10.209upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1357 insertions, 0 deletions

diff --git a/arch/parisc/kernel/pacache.S b/arch/parisc/kernel/pacache.S
new file mode 100644
index 000000000..b2ba6d633
--- /dev/null
+++ b/arch/parisc/kernel/pacache.S
@@ -0,0 +1,1357 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ *  PARISC TLB and cache flushing support
+ *  Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
+ *  Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
+ *  Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
+ */
+
+/*
+ * NOTE: fdc,fic, and pdc instructions that use base register modification
+ *       should only use index and base registers that are not shadowed,
+ *       so that the fast path emulation in the non access miss handler
+ *       can be used.
+ */
+
+#ifdef CONFIG_64BIT
+	.level	2.0w
+#else
+	.level	2.0
+#endif
+
+#include <asm/psw.h>
+#include <asm/assembly.h>
+#include <asm/cache.h>
+#include <asm/ldcw.h>
+#include <asm/alternative.h>
+#include <linux/linkage.h>
+#include <linux/init.h>
+#include <linux/pgtable.h>
+
+	.section .text.hot
+	.align	16
+
+ENTRY_CFI(flush_tlb_all_local)
+	/*
+	 * The pitlbe and pdtlbe instructions should only be used to
+	 * flush the entire tlb. Also, there needs to be no intervening
+	 * tlb operations, e.g. tlb misses, so the operation needs
+	 * to happen in real mode with all interruptions disabled.
+	 */
+
+	/* pcxt_ssm_bug	- relied upon translation! PA 2.0 Arch. F-4 and F-5 */
+	rsm		PSW_SM_I, %r19		/* save I-bit state */
+	load32		PA(1f), %r1
+	nop
+	nop
+	nop
+	nop
+	nop
+
+	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
+	mtctl		%r0, %cr17		/* Clear IIASQ tail */
+	mtctl		%r0, %cr17		/* Clear IIASQ head */
+	mtctl		%r1, %cr18		/* IIAOQ head */
+	ldo		4(%r1), %r1
+	mtctl		%r1, %cr18		/* IIAOQ tail */
+	load32		REAL_MODE_PSW, %r1
+	mtctl           %r1, %ipsw
+	rfi
+	nop
+
+1:      load32		PA(cache_info), %r1
+
+	/* Flush Instruction Tlb */
+
+88:	LDREG		ITLB_SID_BASE(%r1), %r20
+	LDREG		ITLB_SID_STRIDE(%r1), %r21
+	LDREG		ITLB_SID_COUNT(%r1), %r22
+	LDREG		ITLB_OFF_BASE(%r1), %arg0
+	LDREG		ITLB_OFF_STRIDE(%r1), %arg1
+	LDREG		ITLB_OFF_COUNT(%r1), %arg2
+	LDREG		ITLB_LOOP(%r1), %arg3
+
+	addib,COND(=)		-1, %arg3, fitoneloop	/* Preadjust and test */
+	movb,<,n	%arg3, %r31, fitdone	/* If loop < 0, skip */
+	copy		%arg0, %r28		/* Init base addr */
+
+fitmanyloop:					/* Loop if LOOP >= 2 */
+	mtsp		%r20, %sr1
+	add		%r21, %r20, %r20	/* increment space */
+	copy		%arg2, %r29		/* Init middle loop count */
+
+fitmanymiddle:					/* Loop if LOOP >= 2 */
+	addib,COND(>)		-1, %r31, fitmanymiddle	/* Adjusted inner loop decr */
+	pitlbe		%r0(%sr1, %r28)
+	pitlbe,m	%arg1(%sr1, %r28)	/* Last pitlbe and addr adjust */
+	addib,COND(>)		-1, %r29, fitmanymiddle	/* Middle loop decr */
+	copy		%arg3, %r31		/* Re-init inner loop count */
+
+	movb,tr		%arg0, %r28, fitmanyloop /* Re-init base addr */
+	addib,COND(<=),n	-1, %r22, fitdone	/* Outer loop count decr */
+
+fitoneloop:					/* Loop if LOOP = 1 */
+	mtsp		%r20, %sr1
+	copy		%arg0, %r28		/* init base addr */
+	copy		%arg2, %r29		/* init middle loop count */
+
+fitonemiddle:					/* Loop if LOOP = 1 */
+	addib,COND(>)		-1, %r29, fitonemiddle	/* Middle loop count decr */
+	pitlbe,m	%arg1(%sr1, %r28)	/* pitlbe for one loop */
+
+	addib,COND(>)		-1, %r22, fitoneloop	/* Outer loop count decr */
+	add		%r21, %r20, %r20		/* increment space */
+
+fitdone:
+	ALTERNATIVE(88b, fitdone, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
+
+	/* Flush Data Tlb */
+
+	LDREG		DTLB_SID_BASE(%r1), %r20
+	LDREG		DTLB_SID_STRIDE(%r1), %r21
+	LDREG		DTLB_SID_COUNT(%r1), %r22
+	LDREG		DTLB_OFF_BASE(%r1), %arg0
+	LDREG		DTLB_OFF_STRIDE(%r1), %arg1
+	LDREG		DTLB_OFF_COUNT(%r1), %arg2
+	LDREG		DTLB_LOOP(%r1), %arg3
+
+	addib,COND(=)		-1, %arg3, fdtoneloop	/* Preadjust and test */
+	movb,<,n	%arg3, %r31, fdtdone	/* If loop < 0, skip */
+	copy		%arg0, %r28		/* Init base addr */
+
+fdtmanyloop:					/* Loop if LOOP >= 2 */
+	mtsp		%r20, %sr1
+	add		%r21, %r20, %r20	/* increment space */
+	copy		%arg2, %r29		/* Init middle loop count */
+
+fdtmanymiddle:					/* Loop if LOOP >= 2 */
+	addib,COND(>)		-1, %r31, fdtmanymiddle	/* Adjusted inner loop decr */
+	pdtlbe		%r0(%sr1, %r28)
+	pdtlbe,m	%arg1(%sr1, %r28)	/* Last pdtlbe and addr adjust */
+	addib,COND(>)		-1, %r29, fdtmanymiddle	/* Middle loop decr */
+	copy		%arg3, %r31		/* Re-init inner loop count */
+
+	movb,tr		%arg0, %r28, fdtmanyloop /* Re-init base addr */
+	addib,COND(<=),n	-1, %r22,fdtdone	/* Outer loop count decr */
+
+fdtoneloop:					/* Loop if LOOP = 1 */
+	mtsp		%r20, %sr1
+	copy		%arg0, %r28		/* init base addr */
+	copy		%arg2, %r29		/* init middle loop count */
+
+fdtonemiddle:					/* Loop if LOOP = 1 */
+	addib,COND(>)		-1, %r29, fdtonemiddle	/* Middle loop count decr */
+	pdtlbe,m	%arg1(%sr1, %r28)	/* pdtlbe for one loop */
+
+	addib,COND(>)		-1, %r22, fdtoneloop	/* Outer loop count decr */
+	add		%r21, %r20, %r20	/* increment space */
+
+
+fdtdone:
+	/*
+	 * Switch back to virtual mode
+	 */
+	/* pcxt_ssm_bug */
+	rsm		PSW_SM_I, %r0
+	load32		2f, %r1
+	nop
+	nop
+	nop
+	nop
+	nop
+
+	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
+	mtctl		%r0, %cr17		/* Clear IIASQ tail */
+	mtctl		%r0, %cr17		/* Clear IIASQ head */
+	mtctl		%r1, %cr18		/* IIAOQ head */
+	ldo		4(%r1), %r1
+	mtctl		%r1, %cr18		/* IIAOQ tail */
+	load32		KERNEL_PSW, %r1
+	or		%r1, %r19, %r1	/* I-bit to state on entry */
+	mtctl		%r1, %ipsw	/* restore I-bit (entire PSW) */
+	rfi
+	nop
+
+2:      bv		%r0(%r2)
+	nop
+
+	/*
+	 * When running in qemu, drop whole flush_tlb_all_local function and
+	 * replace by one pdtlbe instruction, for which QEMU will drop all
+	 * local TLB entries.
+	 */
+3:	pdtlbe		%r0(%sr1,%r0)
+	bv,n		%r0(%r2)
+	ALTERNATIVE_CODE(flush_tlb_all_local, 2, ALT_COND_RUN_ON_QEMU, 3b)
+ENDPROC_CFI(flush_tlb_all_local)
+
+	.import cache_info,data
+
+ENTRY_CFI(flush_instruction_cache_local)
+88:	load32		cache_info, %r1
+
+	/* Flush Instruction Cache */
+
+	LDREG		ICACHE_BASE(%r1), %arg0
+	LDREG		ICACHE_STRIDE(%r1), %arg1
+	LDREG		ICACHE_COUNT(%r1), %arg2
+	LDREG		ICACHE_LOOP(%r1), %arg3
+	rsm		PSW_SM_I, %r22		/* No mmgt ops during loop*/
+	mtsp		%r0, %sr1
+	addib,COND(=)		-1, %arg3, fioneloop	/* Preadjust and test */
+	movb,<,n	%arg3, %r31, fisync	/* If loop < 0, do sync */
+
+fimanyloop:					/* Loop if LOOP >= 2 */
+	addib,COND(>)		-1, %r31, fimanyloop	/* Adjusted inner loop decr */
+	fice            %r0(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)	/* Last fice and addr adjust */
+	movb,tr		%arg3, %r31, fimanyloop	/* Re-init inner loop count */
+	addib,COND(<=),n	-1, %arg2, fisync	/* Outer loop decr */
+
+fioneloop:					/* Loop if LOOP = 1 */
+	/* Some implementations may flush with a single fice instruction */
+	cmpib,COND(>>=),n	15, %arg2, fioneloop2
+
+fioneloop1:
+	fice,m		%arg1(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)
+	fice,m		%arg1(%sr1, %arg0)
+	addib,COND(>)	-16, %arg2, fioneloop1
+	fice,m		%arg1(%sr1, %arg0)
+
+	/* Check if done */
+	cmpb,COND(=),n	%arg2, %r0, fisync	/* Predict branch taken */
+
+fioneloop2:
+	addib,COND(>)	-1, %arg2, fioneloop2	/* Outer loop count decr */
+	fice,m		%arg1(%sr1, %arg0)	/* Fice for one loop */
+
+fisync:
+	sync
+	mtsm		%r22			/* restore I-bit */
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(flush_instruction_cache_local)
+
+
+	.import cache_info, data
+ENTRY_CFI(flush_data_cache_local)
+88:	load32		cache_info, %r1
+
+	/* Flush Data Cache */
+
+	LDREG		DCACHE_BASE(%r1), %arg0
+	LDREG		DCACHE_STRIDE(%r1), %arg1
+	LDREG		DCACHE_COUNT(%r1), %arg2
+	LDREG		DCACHE_LOOP(%r1), %arg3
+	rsm		PSW_SM_I, %r22		/* No mmgt ops during loop*/
+	mtsp		%r0, %sr1
+	addib,COND(=)		-1, %arg3, fdoneloop	/* Preadjust and test */
+	movb,<,n	%arg3, %r31, fdsync	/* If loop < 0, do sync */
+
+fdmanyloop:					/* Loop if LOOP >= 2 */
+	addib,COND(>)		-1, %r31, fdmanyloop	/* Adjusted inner loop decr */
+	fdce		%r0(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)	/* Last fdce and addr adjust */
+	movb,tr		%arg3, %r31, fdmanyloop	/* Re-init inner loop count */
+	addib,COND(<=),n	-1, %arg2, fdsync	/* Outer loop decr */
+
+fdoneloop:					/* Loop if LOOP = 1 */
+	/* Some implementations may flush with a single fdce instruction */
+	cmpib,COND(>>=),n	15, %arg2, fdoneloop2
+
+fdoneloop1:
+	fdce,m		%arg1(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)
+	fdce,m		%arg1(%sr1, %arg0)
+	addib,COND(>)	-16, %arg2, fdoneloop1
+	fdce,m		%arg1(%sr1, %arg0)
+
+	/* Check if done */
+	cmpb,COND(=),n	%arg2, %r0, fdsync	/* Predict branch taken */
+
+fdoneloop2:
+	addib,COND(>)	-1, %arg2, fdoneloop2	/* Outer loop count decr */
+	fdce,m		%arg1(%sr1, %arg0)	/* Fdce for one loop */
+
+fdsync:
+	syncdma
+	sync
+	mtsm		%r22			/* restore I-bit */
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(flush_data_cache_local)
+
+/* Clear page using kernel mapping.  */
+
+ENTRY_CFI(clear_page_asm)
+#ifdef CONFIG_64BIT
+
+	/* Unroll the loop.  */
+	ldi		(PAGE_SIZE / 128), %r1
+
+1:
+	std		%r0, 0(%r26)
+	std		%r0, 8(%r26)
+	std		%r0, 16(%r26)
+	std		%r0, 24(%r26)
+	std		%r0, 32(%r26)
+	std		%r0, 40(%r26)
+	std		%r0, 48(%r26)
+	std		%r0, 56(%r26)
+	std		%r0, 64(%r26)
+	std		%r0, 72(%r26)
+	std		%r0, 80(%r26)
+	std		%r0, 88(%r26)
+	std		%r0, 96(%r26)
+	std		%r0, 104(%r26)
+	std		%r0, 112(%r26)
+	std		%r0, 120(%r26)
+
+	/* Note reverse branch hint for addib is taken.  */
+	addib,COND(>),n	-1, %r1, 1b
+	ldo		128(%r26), %r26
+
+#else
+
+	/*
+	 * Note that until (if) we start saving the full 64-bit register
+	 * values on interrupt, we can't use std on a 32 bit kernel.
+	 */
+	ldi		(PAGE_SIZE / 64), %r1
+
+1:
+	stw		%r0, 0(%r26)
+	stw		%r0, 4(%r26)
+	stw		%r0, 8(%r26)
+	stw		%r0, 12(%r26)
+	stw		%r0, 16(%r26)
+	stw		%r0, 20(%r26)
+	stw		%r0, 24(%r26)
+	stw		%r0, 28(%r26)
+	stw		%r0, 32(%r26)
+	stw		%r0, 36(%r26)
+	stw		%r0, 40(%r26)
+	stw		%r0, 44(%r26)
+	stw		%r0, 48(%r26)
+	stw		%r0, 52(%r26)
+	stw		%r0, 56(%r26)
+	stw		%r0, 60(%r26)
+
+	addib,COND(>),n	-1, %r1, 1b
+	ldo		64(%r26), %r26
+#endif
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(clear_page_asm)
+
+/* Copy page using kernel mapping.  */
+
+ENTRY_CFI(copy_page_asm)
+#ifdef CONFIG_64BIT
+	/* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
+	 * Unroll the loop by hand and arrange insn appropriately.
+	 * Prefetch doesn't improve performance on rp3440.
+	 * GCC probably can do this just as well...
+	 */
+
+	ldi		(PAGE_SIZE / 128), %r1
+
+1:	ldd		0(%r25), %r19
+	ldd		8(%r25), %r20
+
+	ldd		16(%r25), %r21
+	ldd		24(%r25), %r22
+	std		%r19, 0(%r26)
+	std		%r20, 8(%r26)
+
+	ldd		32(%r25), %r19
+	ldd		40(%r25), %r20
+	std		%r21, 16(%r26)
+	std		%r22, 24(%r26)
+
+	ldd		48(%r25), %r21
+	ldd		56(%r25), %r22
+	std		%r19, 32(%r26)
+	std		%r20, 40(%r26)
+
+	ldd		64(%r25), %r19
+	ldd		72(%r25), %r20
+	std		%r21, 48(%r26)
+	std		%r22, 56(%r26)
+
+	ldd		80(%r25), %r21
+	ldd		88(%r25), %r22
+	std		%r19, 64(%r26)
+	std		%r20, 72(%r26)
+
+	ldd		 96(%r25), %r19
+	ldd		104(%r25), %r20
+	std		%r21, 80(%r26)
+	std		%r22, 88(%r26)
+
+	ldd		112(%r25), %r21
+	ldd		120(%r25), %r22
+	ldo		128(%r25), %r25
+	std		%r19, 96(%r26)
+	std		%r20, 104(%r26)
+
+	std		%r21, 112(%r26)
+	std		%r22, 120(%r26)
+
+	/* Note reverse branch hint for addib is taken.  */
+	addib,COND(>),n	-1, %r1, 1b
+	ldo		128(%r26), %r26
+
+#else
+
+	/*
+	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
+	 * bundles (very restricted rules for bundling).
+	 * Note that until (if) we start saving
+	 * the full 64 bit register values on interrupt, we can't
+	 * use ldd/std on a 32 bit kernel.
+	 */
+	ldw		0(%r25), %r19
+	ldi		(PAGE_SIZE / 64), %r1
+
+1:
+	ldw		4(%r25), %r20
+	ldw		8(%r25), %r21
+	ldw		12(%r25), %r22
+	stw		%r19, 0(%r26)
+	stw		%r20, 4(%r26)
+	stw		%r21, 8(%r26)
+	stw		%r22, 12(%r26)
+	ldw		16(%r25), %r19
+	ldw		20(%r25), %r20
+	ldw		24(%r25), %r21
+	ldw		28(%r25), %r22
+	stw		%r19, 16(%r26)
+	stw		%r20, 20(%r26)
+	stw		%r21, 24(%r26)
+	stw		%r22, 28(%r26)
+	ldw		32(%r25), %r19
+	ldw		36(%r25), %r20
+	ldw		40(%r25), %r21
+	ldw		44(%r25), %r22
+	stw		%r19, 32(%r26)
+	stw		%r20, 36(%r26)
+	stw		%r21, 40(%r26)
+	stw		%r22, 44(%r26)
+	ldw		48(%r25), %r19
+	ldw		52(%r25), %r20
+	ldw		56(%r25), %r21
+	ldw		60(%r25), %r22
+	stw		%r19, 48(%r26)
+	stw		%r20, 52(%r26)
+	ldo		64(%r25), %r25
+	stw		%r21, 56(%r26)
+	stw		%r22, 60(%r26)
+	ldo		64(%r26), %r26
+	addib,COND(>),n	-1, %r1, 1b
+	ldw		0(%r25), %r19
+#endif
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(copy_page_asm)
+
+/*
+ * NOTE: Code in clear_user_page has a hard coded dependency on the
+ *       maximum alias boundary being 4 Mb. We've been assured by the
+ *       parisc chip designers that there will not ever be a parisc
+ *       chip with a larger alias boundary (Never say never :-) ).
+ *
+ *       Subtle: the dtlb miss handlers support the temp alias region by
+ *       "knowing" that if a dtlb miss happens within the temp alias
+ *       region it must have occurred while in clear_user_page. Since
+ *       this routine makes use of processor local translations, we
+ *       don't want to insert them into the kernel page table. Instead,
+ *       we load up some general registers (they need to be registers
+ *       which aren't shadowed) with the physical page numbers (preshifted
+ *       for tlb insertion) needed to insert the translations. When we
+ *       miss on the translation, the dtlb miss handler inserts the
+ *       translation into the tlb using these values:
+ *
+ *          %r26 physical page (shifted for tlb insert) of "to" translation
+ *          %r23 physical page (shifted for tlb insert) of "from" translation
+ */
+
+        /* Drop prot bits and convert to page addr for iitlbt and idtlbt */
+        #define PAGE_ADD_SHIFT  (PAGE_SHIFT-12)
+        .macro          convert_phys_for_tlb_insert20  phys
+        extrd,u         \phys, 56-PAGE_ADD_SHIFT, 32-PAGE_ADD_SHIFT, \phys
+#if _PAGE_SIZE_ENCODING_DEFAULT
+        depdi           _PAGE_SIZE_ENCODING_DEFAULT, 63, (63-58), \phys
+#endif
+	.endm
+
+	/*
+	 * copy_user_page_asm() performs a page copy using mappings
+	 * equivalent to the user page mappings.  It can be used to
+	 * implement copy_user_page() but unfortunately both the `from'
+	 * and `to' pages need to be flushed through mappings equivalent
+	 * to the user mappings after the copy because the kernel accesses
+	 * the `from' page through the kmap kernel mapping and the `to'
+	 * page needs to be flushed since code can be copied.  As a
+	 * result, this implementation is less efficient than the simpler
+	 * copy using the kernel mapping.  It only needs the `from' page
+	 * to flushed via the user mapping.  The kunmap routines handle
+	 * the flushes needed for the kernel mapping.
+	 *
+	 * I'm still keeping this around because it may be possible to
+	 * use it if more information is passed into copy_user_page().
+	 * Have to do some measurements to see if it is worthwhile to
+	 * lobby for such a change.
+	 *
+	 */
+
+ENTRY_CFI(copy_user_page_asm)
+	/* Convert virtual `to' and `from' addresses to physical addresses.
+	   Move `from' physical address to non shadowed register.  */
+	ldil		L%(__PAGE_OFFSET), %r1
+	sub		%r26, %r1, %r26
+	sub		%r25, %r1, %r23
+
+	ldil		L%(TMPALIAS_MAP_START), %r28
+#ifdef CONFIG_64BIT
+#if (TMPALIAS_MAP_START >= 0x80000000)
+	depdi		0, 31,32, %r28		/* clear any sign extension */
+#endif
+	convert_phys_for_tlb_insert20 %r26	/* convert phys addr to tlb insert format */
+	convert_phys_for_tlb_insert20 %r23	/* convert phys addr to tlb insert format */
+	depd		%r24,63,22, %r28	/* Form aliased virtual address 'to' */
+	depdi		0, 63,PAGE_SHIFT, %r28	/* Clear any offset bits */
+	copy		%r28, %r29
+	depdi		1, 41,1, %r29		/* Form aliased virtual address 'from' */
+#else
+	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
+	extrw,u		%r23, 24,25, %r23	/* convert phys addr to tlb insert format */
+	depw		%r24, 31,22, %r28	/* Form aliased virtual address 'to' */
+	depwi		0, 31,PAGE_SHIFT, %r28	/* Clear any offset bits */
+	copy		%r28, %r29
+	depwi		1, 9,1, %r29		/* Form aliased virtual address 'from' */
+#endif
+
+	/* Purge any old translations */
+
+#ifdef CONFIG_PA20
+	pdtlb,l		%r0(%r28)
+	pdtlb,l		%r0(%r29)
+#else
+0:	pdtlb		%r0(%r28)
+1:	pdtlb		%r0(%r29)
+	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
+	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SMP, INSN_PxTLB)
+#endif
+
+#ifdef CONFIG_64BIT
+	/* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
+	 * Unroll the loop by hand and arrange insn appropriately.
+	 * GCC probably can do this just as well.
+	 */
+
+	ldd		0(%r29), %r19
+	ldi		(PAGE_SIZE / 128), %r1
+
+1:	ldd		8(%r29), %r20
+
+	ldd		16(%r29), %r21
+	ldd		24(%r29), %r22
+	std		%r19, 0(%r28)
+	std		%r20, 8(%r28)
+
+	ldd		32(%r29), %r19
+	ldd		40(%r29), %r20
+	std		%r21, 16(%r28)
+	std		%r22, 24(%r28)
+
+	ldd		48(%r29), %r21
+	ldd		56(%r29), %r22
+	std		%r19, 32(%r28)
+	std		%r20, 40(%r28)
+
+	ldd		64(%r29), %r19
+	ldd		72(%r29), %r20
+	std		%r21, 48(%r28)
+	std		%r22, 56(%r28)
+
+	ldd		80(%r29), %r21
+	ldd		88(%r29), %r22
+	std		%r19, 64(%r28)
+	std		%r20, 72(%r28)
+
+	ldd		 96(%r29), %r19
+	ldd		104(%r29), %r20
+	std		%r21, 80(%r28)
+	std		%r22, 88(%r28)
+
+	ldd		112(%r29), %r21
+	ldd		120(%r29), %r22
+	std		%r19, 96(%r28)
+	std		%r20, 104(%r28)
+
+	ldo		128(%r29), %r29
+	std		%r21, 112(%r28)
+	std		%r22, 120(%r28)
+	ldo		128(%r28), %r28
+
+	/* conditional branches nullify on forward taken branch, and on
+	 * non-taken backward branch. Note that .+4 is a backwards branch.
+	 * The ldd should only get executed if the branch is taken.
+	 */
+	addib,COND(>),n	-1, %r1, 1b		/* bundle 10 */
+	ldd		0(%r29), %r19		/* start next loads */
+
+#else
+	ldi		(PAGE_SIZE / 64), %r1
+
+	/*
+	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
+	 * bundles (very restricted rules for bundling). It probably
+	 * does OK on PCXU and better, but we could do better with
+	 * ldd/std instructions. Note that until (if) we start saving
+	 * the full 64 bit register values on interrupt, we can't
+	 * use ldd/std on a 32 bit kernel.
+	 */
+
+1:	ldw		0(%r29), %r19
+	ldw		4(%r29), %r20
+	ldw		8(%r29), %r21
+	ldw		12(%r29), %r22
+	stw		%r19, 0(%r28)
+	stw		%r20, 4(%r28)
+	stw		%r21, 8(%r28)
+	stw		%r22, 12(%r28)
+	ldw		16(%r29), %r19
+	ldw		20(%r29), %r20
+	ldw		24(%r29), %r21
+	ldw		28(%r29), %r22
+	stw		%r19, 16(%r28)
+	stw		%r20, 20(%r28)
+	stw		%r21, 24(%r28)
+	stw		%r22, 28(%r28)
+	ldw		32(%r29), %r19
+	ldw		36(%r29), %r20
+	ldw		40(%r29), %r21
+	ldw		44(%r29), %r22
+	stw		%r19, 32(%r28)
+	stw		%r20, 36(%r28)
+	stw		%r21, 40(%r28)
+	stw		%r22, 44(%r28)
+	ldw		48(%r29), %r19
+	ldw		52(%r29), %r20
+	ldw		56(%r29), %r21
+	ldw		60(%r29), %r22
+	stw		%r19, 48(%r28)
+	stw		%r20, 52(%r28)
+	stw		%r21, 56(%r28)
+	stw		%r22, 60(%r28)
+	ldo		64(%r28), %r28
+
+	addib,COND(>)		-1, %r1,1b
+	ldo		64(%r29), %r29
+#endif
+
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(copy_user_page_asm)
+
+ENTRY_CFI(clear_user_page_asm)
+	tophys_r1	%r26
+
+	ldil		L%(TMPALIAS_MAP_START), %r28
+#ifdef CONFIG_64BIT
+#if (TMPALIAS_MAP_START >= 0x80000000)
+	depdi		0, 31,32, %r28		/* clear any sign extension */
+#endif
+	convert_phys_for_tlb_insert20 %r26	/* convert phys addr to tlb insert format */
+	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
+	depdi		0, 63,PAGE_SHIFT, %r28	/* Clear any offset bits */
+#else
+	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
+	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
+	depwi		0, 31,PAGE_SHIFT, %r28	/* Clear any offset bits */
+#endif
+
+	/* Purge any old translation */
+
+#ifdef CONFIG_PA20
+	pdtlb,l		%r0(%r28)
+#else
+0:	pdtlb		%r0(%r28)
+	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
+#endif
+
+#ifdef CONFIG_64BIT
+	ldi		(PAGE_SIZE / 128), %r1
+
+	/* PREFETCH (Write) has not (yet) been proven to help here */
+	/* #define	PREFETCHW_OP	ldd		256(%0), %r0 */
+
+1:	std		%r0, 0(%r28)
+	std		%r0, 8(%r28)
+	std		%r0, 16(%r28)
+	std		%r0, 24(%r28)
+	std		%r0, 32(%r28)
+	std		%r0, 40(%r28)
+	std		%r0, 48(%r28)
+	std		%r0, 56(%r28)
+	std		%r0, 64(%r28)
+	std		%r0, 72(%r28)
+	std		%r0, 80(%r28)
+	std		%r0, 88(%r28)
+	std		%r0, 96(%r28)
+	std		%r0, 104(%r28)
+	std		%r0, 112(%r28)
+	std		%r0, 120(%r28)
+	addib,COND(>)		-1, %r1, 1b
+	ldo		128(%r28), %r28
+
+#else	/* ! CONFIG_64BIT */
+	ldi		(PAGE_SIZE / 64), %r1
+
+1:	stw		%r0, 0(%r28)
+	stw		%r0, 4(%r28)
+	stw		%r0, 8(%r28)
+	stw		%r0, 12(%r28)
+	stw		%r0, 16(%r28)
+	stw		%r0, 20(%r28)
+	stw		%r0, 24(%r28)
+	stw		%r0, 28(%r28)
+	stw		%r0, 32(%r28)
+	stw		%r0, 36(%r28)
+	stw		%r0, 40(%r28)
+	stw		%r0, 44(%r28)
+	stw		%r0, 48(%r28)
+	stw		%r0, 52(%r28)
+	stw		%r0, 56(%r28)
+	stw		%r0, 60(%r28)
+	addib,COND(>)		-1, %r1, 1b
+	ldo		64(%r28), %r28
+#endif	/* CONFIG_64BIT */
+
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(clear_user_page_asm)
+
+ENTRY_CFI(flush_dcache_page_asm)
+	ldil		L%(TMPALIAS_MAP_START), %r28
+#ifdef CONFIG_64BIT
+#if (TMPALIAS_MAP_START >= 0x80000000)
+	depdi		0, 31,32, %r28		/* clear any sign extension */
+#endif
+	convert_phys_for_tlb_insert20 %r26	/* convert phys addr to tlb insert format */
+	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
+	depdi		0, 63,PAGE_SHIFT, %r28	/* Clear any offset bits */
+#else
+	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
+	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
+	depwi		0, 31,PAGE_SHIFT, %r28	/* Clear any offset bits */
+#endif
+
+	/* Purge any old translation */
+
+#ifdef CONFIG_PA20
+	pdtlb,l		%r0(%r28)
+#else
+0:	pdtlb		%r0(%r28)
+	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
+#endif
+
+88:	ldil		L%dcache_stride, %r1
+	ldw		R%dcache_stride(%r1), r31
+
+#ifdef CONFIG_64BIT
+	depdi,z		1, 63-PAGE_SHIFT,1, %r25
+#else
+	depwi,z		1, 31-PAGE_SHIFT,1, %r25
+#endif
+	add		%r28, %r25, %r25
+	sub		%r25, r31, %r25
+
+1:	fdc,m		r31(%r28)
+	fdc,m		r31(%r28)
+	fdc,m		r31(%r28)
+	fdc,m		r31(%r28)
+	fdc,m		r31(%r28)
+	fdc,m		r31(%r28)
+	fdc,m		r31(%r28)
+	fdc,m		r31(%r28)
+	fdc,m		r31(%r28)
+	fdc,m		r31(%r28)
+	fdc,m		r31(%r28)
+	fdc,m		r31(%r28)
+	fdc,m		r31(%r28)
+	fdc,m		r31(%r28)
+	fdc,m		r31(%r28)
+	cmpb,COND(>>)	%r25, %r28, 1b /* predict taken */
+	fdc,m		r31(%r28)
+
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
+	sync
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(flush_dcache_page_asm)
+
+ENTRY_CFI(purge_dcache_page_asm)
+	ldil		L%(TMPALIAS_MAP_START), %r28
+#ifdef CONFIG_64BIT
+#if (TMPALIAS_MAP_START >= 0x80000000)
+	depdi		0, 31,32, %r28		/* clear any sign extension */
+#endif
+	convert_phys_for_tlb_insert20 %r26	/* convert phys addr to tlb insert format */
+	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
+	depdi		0, 63,PAGE_SHIFT, %r28	/* Clear any offset bits */
+#else
+	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
+	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
+	depwi		0, 31,PAGE_SHIFT, %r28	/* Clear any offset bits */
+#endif
+
+	/* Purge any old translation */
+
+#ifdef CONFIG_PA20
+	pdtlb,l		%r0(%r28)
+#else
+0:	pdtlb		%r0(%r28)
+	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
+#endif
+
+88:	ldil		L%dcache_stride, %r1
+	ldw		R%dcache_stride(%r1), r31
+
+#ifdef CONFIG_64BIT
+	depdi,z		1, 63-PAGE_SHIFT,1, %r25
+#else
+	depwi,z		1, 31-PAGE_SHIFT,1, %r25
+#endif
+	add		%r28, %r25, %r25
+	sub		%r25, r31, %r25
+
+1:      pdc,m		r31(%r28)
+	pdc,m		r31(%r28)
+	pdc,m		r31(%r28)
+	pdc,m		r31(%r28)
+	pdc,m		r31(%r28)
+	pdc,m		r31(%r28)
+	pdc,m		r31(%r28)
+	pdc,m		r31(%r28)
+	pdc,m		r31(%r28)
+	pdc,m		r31(%r28)
+	pdc,m		r31(%r28)
+	pdc,m		r31(%r28)
+	pdc,m		r31(%r28)
+	pdc,m		r31(%r28)
+	pdc,m		r31(%r28)
+	cmpb,COND(>>)	%r25, %r28, 1b /* predict taken */
+	pdc,m		r31(%r28)
+
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
+	sync
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(purge_dcache_page_asm)
+
+ENTRY_CFI(flush_icache_page_asm)
+	ldil		L%(TMPALIAS_MAP_START), %r28
+#ifdef CONFIG_64BIT
+#if (TMPALIAS_MAP_START >= 0x80000000)
+	depdi		0, 31,32, %r28		/* clear any sign extension */
+#endif
+	convert_phys_for_tlb_insert20 %r26	/* convert phys addr to tlb insert format */
+	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
+	depdi		0, 63,PAGE_SHIFT, %r28	/* Clear any offset bits */
+#else
+	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
+	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
+	depwi		0, 31,PAGE_SHIFT, %r28	/* Clear any offset bits */
+#endif
+
+	/* Purge any old translation.  Note that the FIC instruction
+	 * may use either the instruction or data TLB.  Given that we
+	 * have a flat address space, it's not clear which TLB will be
+	 * used.  So, we purge both entries.  */
+
+#ifdef CONFIG_PA20
+	pdtlb,l		%r0(%r28)
+1:	pitlb,l         %r0(%sr4,%r28)
+	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
+#else
+0:	pdtlb		%r0(%r28)
+1:	pitlb           %r0(%sr4,%r28)
+	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
+	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SMP, INSN_PxTLB)
+	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
+#endif
+
+88:	ldil		L%icache_stride, %r1
+	ldw		R%icache_stride(%r1), %r31
+
+#ifdef CONFIG_64BIT
+	depdi,z		1, 63-PAGE_SHIFT,1, %r25
+#else
+	depwi,z		1, 31-PAGE_SHIFT,1, %r25
+#endif
+	add		%r28, %r25, %r25
+	sub		%r25, %r31, %r25
+
+	/* fic only has the type 26 form on PA1.1, requiring an
+	 * explicit space specification, so use %sr4 */
+1:      fic,m		%r31(%sr4,%r28)
+	fic,m		%r31(%sr4,%r28)
+	fic,m		%r31(%sr4,%r28)
+	fic,m		%r31(%sr4,%r28)
+	fic,m		%r31(%sr4,%r28)
+	fic,m		%r31(%sr4,%r28)
+	fic,m		%r31(%sr4,%r28)
+	fic,m		%r31(%sr4,%r28)
+	fic,m		%r31(%sr4,%r28)
+	fic,m		%r31(%sr4,%r28)
+	fic,m		%r31(%sr4,%r28)
+	fic,m		%r31(%sr4,%r28)
+	fic,m		%r31(%sr4,%r28)
+	fic,m		%r31(%sr4,%r28)
+	fic,m		%r31(%sr4,%r28)
+	cmpb,COND(>>)	%r25, %r28, 1b /* predict taken */
+	fic,m		%r31(%sr4,%r28)
+
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
+	sync
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(flush_icache_page_asm)
+
+ENTRY_CFI(flush_kernel_dcache_page_asm)
+88:	ldil		L%dcache_stride, %r1
+	ldw		R%dcache_stride(%r1), %r23
+
+#ifdef CONFIG_64BIT
+	depdi,z		1, 63-PAGE_SHIFT,1, %r25
+#else
+	depwi,z		1, 31-PAGE_SHIFT,1, %r25
+#endif
+	add		%r26, %r25, %r25
+	sub		%r25, %r23, %r25
+
+1:      fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	cmpb,COND(>>)	%r25, %r26, 1b /* predict taken */
+	fdc,m		%r23(%r26)
+
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
+	sync
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(flush_kernel_dcache_page_asm)
+
+ENTRY_CFI(purge_kernel_dcache_page_asm)
+88:	ldil		L%dcache_stride, %r1
+	ldw		R%dcache_stride(%r1), %r23
+
+#ifdef CONFIG_64BIT
+	depdi,z		1, 63-PAGE_SHIFT,1, %r25
+#else
+	depwi,z		1, 31-PAGE_SHIFT,1, %r25
+#endif
+	add		%r26, %r25, %r25
+	sub		%r25, %r23, %r25
+
+1:      pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	cmpb,COND(>>)	%r25, %r26, 1b /* predict taken */
+	pdc,m		%r23(%r26)
+
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
+	sync
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(purge_kernel_dcache_page_asm)
+
+ENTRY_CFI(flush_user_dcache_range_asm)
+88:	ldil		L%dcache_stride, %r1
+	ldw		R%dcache_stride(%r1), %r23
+	ldo		-1(%r23), %r21
+	ANDCM		%r26, %r21, %r26
+
+#ifdef CONFIG_64BIT
+	depd,z		%r23, 59, 60, %r21
+#else
+	depw,z		%r23, 27, 28, %r21
+#endif
+	add		%r26, %r21, %r22
+	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
+1:	add		%r22, %r21, %r22
+	fdc,m		%r23(%sr3, %r26)
+	fdc,m		%r23(%sr3, %r26)
+	fdc,m		%r23(%sr3, %r26)
+	fdc,m		%r23(%sr3, %r26)
+	fdc,m		%r23(%sr3, %r26)
+	fdc,m		%r23(%sr3, %r26)
+	fdc,m		%r23(%sr3, %r26)
+	fdc,m		%r23(%sr3, %r26)
+	fdc,m		%r23(%sr3, %r26)
+	fdc,m		%r23(%sr3, %r26)
+	fdc,m		%r23(%sr3, %r26)
+	fdc,m		%r23(%sr3, %r26)
+	fdc,m		%r23(%sr3, %r26)
+	fdc,m		%r23(%sr3, %r26)
+	fdc,m		%r23(%sr3, %r26)
+	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
+	fdc,m		%r23(%sr3, %r26)
+
+2:	cmpb,COND(>>),n	%r25, %r26, 2b
+	fdc,m		%r23(%sr3, %r26)
+
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
+	sync
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(flush_user_dcache_range_asm)
+
+ENTRY_CFI(flush_kernel_dcache_range_asm)
+88:	ldil		L%dcache_stride, %r1
+	ldw		R%dcache_stride(%r1), %r23
+	ldo		-1(%r23), %r21
+	ANDCM		%r26, %r21, %r26
+
+#ifdef CONFIG_64BIT
+	depd,z		%r23, 59, 60, %r21
+#else
+	depw,z		%r23, 27, 28, %r21
+#endif
+	add		%r26, %r21, %r22
+	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
+1:	add		%r22, %r21, %r22
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	fdc,m		%r23(%r26)
+	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
+	fdc,m		%r23(%r26)
+
+2:	cmpb,COND(>>),n	%r25, %r26, 2b /* predict taken */
+	fdc,m		%r23(%r26)
+
+	sync
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
+	syncdma
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(flush_kernel_dcache_range_asm)
+
+ENTRY_CFI(purge_kernel_dcache_range_asm)
+88:	ldil		L%dcache_stride, %r1
+	ldw		R%dcache_stride(%r1), %r23
+	ldo		-1(%r23), %r21
+	ANDCM		%r26, %r21, %r26
+
+#ifdef CONFIG_64BIT
+	depd,z		%r23, 59, 60, %r21
+#else
+	depw,z		%r23, 27, 28, %r21
+#endif
+	add		%r26, %r21, %r22
+	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
+1:	add		%r22, %r21, %r22
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	pdc,m		%r23(%r26)
+	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
+	pdc,m		%r23(%r26)
+
+2:	cmpb,COND(>>),n	%r25, %r26, 2b /* predict taken */
+	pdc,m		%r23(%r26)
+
+	sync
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
+	syncdma
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(purge_kernel_dcache_range_asm)
+
+ENTRY_CFI(flush_user_icache_range_asm)
+88:	ldil		L%icache_stride, %r1
+	ldw		R%icache_stride(%r1), %r23
+	ldo		-1(%r23), %r21
+	ANDCM		%r26, %r21, %r26
+
+#ifdef CONFIG_64BIT
+	depd,z		%r23, 59, 60, %r21
+#else
+	depw,z		%r23, 27, 28, %r21
+#endif
+	add		%r26, %r21, %r22
+	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
+1:	add		%r22, %r21, %r22
+	fic,m		%r23(%sr3, %r26)
+	fic,m		%r23(%sr3, %r26)
+	fic,m		%r23(%sr3, %r26)
+	fic,m		%r23(%sr3, %r26)
+	fic,m		%r23(%sr3, %r26)
+	fic,m		%r23(%sr3, %r26)
+	fic,m		%r23(%sr3, %r26)
+	fic,m		%r23(%sr3, %r26)
+	fic,m		%r23(%sr3, %r26)
+	fic,m		%r23(%sr3, %r26)
+	fic,m		%r23(%sr3, %r26)
+	fic,m		%r23(%sr3, %r26)
+	fic,m		%r23(%sr3, %r26)
+	fic,m		%r23(%sr3, %r26)
+	fic,m		%r23(%sr3, %r26)
+	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
+	fic,m		%r23(%sr3, %r26)
+
+2:	cmpb,COND(>>),n	%r25, %r26, 2b
+	fic,m		%r23(%sr3, %r26)
+
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
+	sync
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(flush_user_icache_range_asm)
+
+ENTRY_CFI(flush_kernel_icache_page)
+88:	ldil		L%icache_stride, %r1
+	ldw		R%icache_stride(%r1), %r23
+
+#ifdef CONFIG_64BIT
+	depdi,z		1, 63-PAGE_SHIFT,1, %r25
+#else
+	depwi,z		1, 31-PAGE_SHIFT,1, %r25
+#endif
+	add		%r26, %r25, %r25
+	sub		%r25, %r23, %r25
+
+
+1:      fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	cmpb,COND(>>)	%r25, %r26, 1b /* predict taken */
+	fic,m		%r23(%sr4, %r26)
+
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
+	sync
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(flush_kernel_icache_page)
+
+ENTRY_CFI(flush_kernel_icache_range_asm)
+88:	ldil		L%icache_stride, %r1
+	ldw		R%icache_stride(%r1), %r23
+	ldo		-1(%r23), %r21
+	ANDCM		%r26, %r21, %r26
+
+#ifdef CONFIG_64BIT
+	depd,z		%r23, 59, 60, %r21
+#else
+	depw,z		%r23, 27, 28, %r21
+#endif
+	add		%r26, %r21, %r22
+	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
+1:	add		%r22, %r21, %r22
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	fic,m		%r23(%sr4, %r26)
+	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
+	fic,m		%r23(%sr4, %r26)
+
+2:	cmpb,COND(>>),n	%r25, %r26, 2b /* predict taken */
+	fic,m		%r23(%sr4, %r26)
+
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
+	sync
+	bv		%r0(%r2)
+	nop
+ENDPROC_CFI(flush_kernel_icache_range_asm)
+
+	__INIT
+
+	/* align should cover use of rfi in disable_sr_hashing_asm and
+	 * srdis_done.
+	 */
+	.align	256
+ENTRY_CFI(disable_sr_hashing_asm)
+	/*
+	 * Switch to real mode
+	 */
+	/* pcxt_ssm_bug */
+	rsm		PSW_SM_I, %r0
+	load32		PA(1f), %r1
+	nop
+	nop
+	nop
+	nop
+	nop
+
+	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
+	mtctl		%r0, %cr17		/* Clear IIASQ tail */
+	mtctl		%r0, %cr17		/* Clear IIASQ head */
+	mtctl		%r1, %cr18		/* IIAOQ head */
+	ldo		4(%r1), %r1
+	mtctl		%r1, %cr18		/* IIAOQ tail */
+	load32		REAL_MODE_PSW, %r1
+	mtctl		%r1, %ipsw
+	rfi
+	nop
+
+1:      cmpib,=,n	SRHASH_PCXST, %r26,srdis_pcxs
+	cmpib,=,n	SRHASH_PCXL, %r26,srdis_pcxl
+	cmpib,=,n	SRHASH_PA20, %r26,srdis_pa20
+	b,n		srdis_done
+
+srdis_pcxs:
+
+	/* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
+
+	.word		0x141c1a00		/* mfdiag %dr0, %r28 */
+	.word		0x141c1a00		/* must issue twice */
+	depwi		0,18,1, %r28		/* Clear DHE (dcache hash enable) */
+	depwi		0,20,1, %r28		/* Clear IHE (icache hash enable) */
+	.word		0x141c1600		/* mtdiag %r28, %dr0 */
+	.word		0x141c1600		/* must issue twice */
+	b,n		srdis_done
+
+srdis_pcxl:
+
+	/* Disable Space Register Hashing for PCXL */
+
+	.word		0x141c0600		/* mfdiag %dr0, %r28 */
+	depwi           0,28,2, %r28		/* Clear DHASH_EN & IHASH_EN */
+	.word		0x141c0240		/* mtdiag %r28, %dr0 */
+	b,n		srdis_done
+
+srdis_pa20:
+
+	/* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+,PCXW2 */
+
+	.word		0x144008bc		/* mfdiag %dr2, %r28 */
+	depdi		0, 54,1, %r28		/* clear DIAG_SPHASH_ENAB (bit 54) */
+	.word		0x145c1840		/* mtdiag %r28, %dr2 */
+
+
+srdis_done:
+	/* Switch back to virtual mode */
+	rsm		PSW_SM_I, %r0		/* prep to load iia queue */
+	load32 	   	2f, %r1
+	nop
+	nop
+	nop
+	nop
+	nop
+
+	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
+	mtctl		%r0, %cr17		/* Clear IIASQ tail */
+	mtctl		%r0, %cr17		/* Clear IIASQ head */
+	mtctl		%r1, %cr18		/* IIAOQ head */
+	ldo		4(%r1), %r1
+	mtctl		%r1, %cr18		/* IIAOQ tail */
+	load32		KERNEL_PSW, %r1
+	mtctl		%r1, %ipsw
+	rfi
+	nop
+
+2:      bv		%r0(%r2)
+	nop
+ENDPROC_CFI(disable_sr_hashing_asm)
+
+	.end