summaryrefslogtreecommitdiffstats
path: root/arch/ia64/lib/memcpy.S
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-18 17:40:19 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-18 17:40:19 +0000
commit9f0fc191371843c4fc000a226b0a26b6c059aacd (patch)
tree35f8be3ef04506ac891ad001e8c41e535ae8d01d /arch/ia64/lib/memcpy.S
parentReleasing progress-linux version 6.6.15-2~progress7.99u1. (diff)
downloadlinux-9f0fc191371843c4fc000a226b0a26b6c059aacd.tar.xz
linux-9f0fc191371843c4fc000a226b0a26b6c059aacd.zip
Merging upstream version 6.7.7.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'arch/ia64/lib/memcpy.S')
-rw-r--r--arch/ia64/lib/memcpy.S304
1 files changed, 0 insertions, 304 deletions
diff --git a/arch/ia64/lib/memcpy.S b/arch/ia64/lib/memcpy.S
deleted file mode 100644
index 35c9069a83..0000000000
--- a/arch/ia64/lib/memcpy.S
+++ /dev/null
@@ -1,304 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- *
- * Optimized version of the standard memcpy() function
- *
- * Inputs:
- * in0: destination address
- * in1: source address
- * in2: number of bytes to copy
- * Output:
- * no return value
- *
- * Copyright (C) 2000-2001 Hewlett-Packard Co
- * Stephane Eranian <eranian@hpl.hp.com>
- * David Mosberger-Tang <davidm@hpl.hp.com>
- */
-#include <linux/export.h>
-#include <asm/asmmacro.h>
-
-GLOBAL_ENTRY(memcpy)
-
-# define MEM_LAT 21 /* latency to memory */
-
-# define dst r2
-# define src r3
-# define retval r8
-# define saved_pfs r9
-# define saved_lc r10
-# define saved_pr r11
-# define cnt r16
-# define src2 r17
-# define t0 r18
-# define t1 r19
-# define t2 r20
-# define t3 r21
-# define t4 r22
-# define src_end r23
-
-# define N (MEM_LAT + 4)
-# define Nrot ((N + 7) & ~7)
-
- /*
- * First, check if everything (src, dst, len) is a multiple of eight. If
- * so, we handle everything with no taken branches (other than the loop
- * itself) and a small icache footprint. Otherwise, we jump off to
- * the more general copy routine handling arbitrary
- * sizes/alignment etc.
- */
- .prologue
- .save ar.pfs, saved_pfs
- alloc saved_pfs=ar.pfs,3,Nrot,0,Nrot
- .save ar.lc, saved_lc
- mov saved_lc=ar.lc
- or t0=in0,in1
- ;;
-
- or t0=t0,in2
- .save pr, saved_pr
- mov saved_pr=pr
-
- .body
-
- cmp.eq p6,p0=in2,r0 // zero length?
- mov retval=in0 // return dst
-(p6) br.ret.spnt.many rp // zero length, return immediately
- ;;
-
- mov dst=in0 // copy because of rotation
- shr.u cnt=in2,3 // number of 8-byte words to copy
- mov pr.rot=1<<16
- ;;
-
- adds cnt=-1,cnt // br.ctop is repeat/until
- cmp.gtu p7,p0=16,in2 // copying less than 16 bytes?
- mov ar.ec=N
- ;;
-
- and t0=0x7,t0
- mov ar.lc=cnt
- ;;
- cmp.ne p6,p0=t0,r0
-
- mov src=in1 // copy because of rotation
-(p7) br.cond.spnt.few .memcpy_short
-(p6) br.cond.spnt.few .memcpy_long
- ;;
- nop.m 0
- ;;
- nop.m 0
- nop.i 0
- ;;
- nop.m 0
- ;;
- .rotr val[N]
- .rotp p[N]
- .align 32
-1: { .mib
-(p[0]) ld8 val[0]=[src],8
- nop.i 0
- brp.loop.imp 1b, 2f
-}
-2: { .mfb
-(p[N-1])st8 [dst]=val[N-1],8
- nop.f 0
- br.ctop.dptk.few 1b
-}
- ;;
- mov ar.lc=saved_lc
- mov pr=saved_pr,-1
- mov ar.pfs=saved_pfs
- br.ret.sptk.many rp
-
- /*
- * Small (<16 bytes) unaligned copying is done via a simple byte-at-the-time
- * copy loop. This performs relatively poorly on Itanium, but it doesn't
- * get used very often (gcc inlines small copies) and due to atomicity
- * issues, we want to avoid read-modify-write of entire words.
- */
- .align 32
-.memcpy_short:
- adds cnt=-1,in2 // br.ctop is repeat/until
- mov ar.ec=MEM_LAT
- brp.loop.imp 1f, 2f
- ;;
- mov ar.lc=cnt
- ;;
- nop.m 0
- ;;
- nop.m 0
- nop.i 0
- ;;
- nop.m 0
- ;;
- nop.m 0
- ;;
- /*
- * It is faster to put a stop bit in the loop here because it makes
- * the pipeline shorter (and latency is what matters on short copies).
- */
- .align 32
-1: { .mib
-(p[0]) ld1 val[0]=[src],1
- nop.i 0
- brp.loop.imp 1b, 2f
-} ;;
-2: { .mfb
-(p[MEM_LAT-1])st1 [dst]=val[MEM_LAT-1],1
- nop.f 0
- br.ctop.dptk.few 1b
-} ;;
- mov ar.lc=saved_lc
- mov pr=saved_pr,-1
- mov ar.pfs=saved_pfs
- br.ret.sptk.many rp
-
- /*
- * Large (>= 16 bytes) copying is done in a fancy way. Latency isn't
- * an overriding concern here, but throughput is. We first do
- * sub-word copying until the destination is aligned, then we check
- * if the source is also aligned. If so, we do a simple load/store-loop
- * until there are less than 8 bytes left over and then we do the tail,
- * by storing the last few bytes using sub-word copying. If the source
- * is not aligned, we branch off to the non-congruent loop.
- *
- * stage: op:
- * 0 ld
- * :
- * MEM_LAT+3 shrp
- * MEM_LAT+4 st
- *
- * On Itanium, the pipeline itself runs without stalls. However, br.ctop
- * seems to introduce an unavoidable bubble in the pipeline so the overall
- * latency is 2 cycles/iteration. This gives us a _copy_ throughput
- * of 4 byte/cycle. Still not bad.
- */
-# undef N
-# undef Nrot
-# define N (MEM_LAT + 5) /* number of stages */
-# define Nrot ((N+1 + 2 + 7) & ~7) /* number of rotating regs */
-
-#define LOG_LOOP_SIZE 6
-
-.memcpy_long:
- alloc t3=ar.pfs,3,Nrot,0,Nrot // resize register frame
- and t0=-8,src // t0 = src & ~7
- and t2=7,src // t2 = src & 7
- ;;
- ld8 t0=[t0] // t0 = 1st source word
- adds src2=7,src // src2 = (src + 7)
- sub t4=r0,dst // t4 = -dst
- ;;
- and src2=-8,src2 // src2 = (src + 7) & ~7
- shl t2=t2,3 // t2 = 8*(src & 7)
- shl t4=t4,3 // t4 = 8*(dst & 7)
- ;;
- ld8 t1=[src2] // t1 = 1st source word if src is 8-byte aligned, 2nd otherwise
- sub t3=64,t2 // t3 = 64-8*(src & 7)
- shr.u t0=t0,t2
- ;;
- add src_end=src,in2
- shl t1=t1,t3
- mov pr=t4,0x38 // (p5,p4,p3)=(dst & 7)
- ;;
- or t0=t0,t1
- mov cnt=r0
- adds src_end=-1,src_end
- ;;
-(p3) st1 [dst]=t0,1
-(p3) shr.u t0=t0,8
-(p3) adds cnt=1,cnt
- ;;
-(p4) st2 [dst]=t0,2
-(p4) shr.u t0=t0,16
-(p4) adds cnt=2,cnt
- ;;
-(p5) st4 [dst]=t0,4
-(p5) adds cnt=4,cnt
- and src_end=-8,src_end // src_end = last word of source buffer
- ;;
-
- // At this point, dst is aligned to 8 bytes and there at least 16-7=9 bytes left to copy:
-
-1:{ add src=cnt,src // make src point to remainder of source buffer
- sub cnt=in2,cnt // cnt = number of bytes left to copy
- mov t4=ip
- } ;;
- and src2=-8,src // align source pointer
- adds t4=.memcpy_loops-1b,t4
- mov ar.ec=N
-
- and t0=7,src // t0 = src & 7
- shr.u t2=cnt,3 // t2 = number of 8-byte words left to copy
- shl cnt=cnt,3 // move bits 0-2 to 3-5
- ;;
-
- .rotr val[N+1], w[2]
- .rotp p[N]
-
- cmp.ne p6,p0=t0,r0 // is src aligned, too?
- shl t0=t0,LOG_LOOP_SIZE // t0 = 8*(src & 7)
- adds t2=-1,t2 // br.ctop is repeat/until
- ;;
- add t4=t0,t4
- mov pr=cnt,0x38 // set (p5,p4,p3) to # of bytes last-word bytes to copy
- mov ar.lc=t2
- ;;
- nop.m 0
- ;;
- nop.m 0
- nop.i 0
- ;;
- nop.m 0
- ;;
-(p6) ld8 val[1]=[src2],8 // prime the pump...
- mov b6=t4
- br.sptk.few b6
- ;;
-
-.memcpy_tail:
- // At this point, (p5,p4,p3) are set to the number of bytes left to copy (which is
- // less than 8) and t0 contains the last few bytes of the src buffer:
-(p5) st4 [dst]=t0,4
-(p5) shr.u t0=t0,32
- mov ar.lc=saved_lc
- ;;
-(p4) st2 [dst]=t0,2
-(p4) shr.u t0=t0,16
- mov ar.pfs=saved_pfs
- ;;
-(p3) st1 [dst]=t0
- mov pr=saved_pr,-1
- br.ret.sptk.many rp
-
-///////////////////////////////////////////////////////
- .align 64
-
-#define COPY(shift,index) \
- 1: { .mib \
- (p[0]) ld8 val[0]=[src2],8; \
- (p[MEM_LAT+3]) shrp w[0]=val[MEM_LAT+3],val[MEM_LAT+4-index],shift; \
- brp.loop.imp 1b, 2f \
- }; \
- 2: { .mfb \
- (p[MEM_LAT+4]) st8 [dst]=w[1],8; \
- nop.f 0; \
- br.ctop.dptk.few 1b; \
- }; \
- ;; \
- ld8 val[N-1]=[src_end]; /* load last word (may be same as val[N]) */ \
- ;; \
- shrp t0=val[N-1],val[N-index],shift; \
- br .memcpy_tail
-.memcpy_loops:
- COPY(0, 1) /* no point special casing this---it doesn't go any faster without shrp */
- COPY(8, 0)
- COPY(16, 0)
- COPY(24, 0)
- COPY(32, 0)
- COPY(40, 0)
- COPY(48, 0)
- COPY(56, 0)
-
-END(memcpy)
-EXPORT_SYMBOL(memcpy)