minix/common/lib/libc/arch/sparc64/string/memcpy.S

1628 lines
31 KiB
ArmAsm
Raw Normal View History

/* $NetBSD: memcpy.S,v 1.2 2013/03/17 02:13:10 christos Exp $ */
/*
* Copyright (c) 1996-2002 Eduardo Horvath
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
#include "strmacros.h"
#if defined(LIBC_SCCS) && !defined(lint)
RCSID("$NetBSD: memcpy.S,v 1.2 2013/03/17 02:13:10 christos Exp $")
#endif /* LIBC_SCCS and not lint */
/*
* memcpy
* Assumes regions do not overlap;
*
* Must not use %g7 (see copyin/copyout above).
*/
ENTRY(memcpy) /* dest, src, size */
/*
* Swap args for bcopy. Gcc generates calls to memcpy for
* structure assignments.
*/
mov %o0, %o3
mov %o1, %o0
mov %o3, %o1
#if !defined(_KERNEL) || defined(_RUMPKERNEL)
ENTRY(bcopy) /* src, dest, size */
#endif
#ifdef DEBUG
#if defined(_KERNEL) && !defined(_RUMPKERNEL)
set pmapdebug, %o4
ld [%o4], %o4
btst 0x80, %o4 ! PDB_COPY
bz,pt %icc, 3f
nop
#endif
save %sp, -CC64FSZ, %sp
mov %i0, %o1
set 2f, %o0
mov %i1, %o2
call printf
mov %i2, %o3
! ta 1; nop
restore
.data
2: .asciz "memcpy(%p<-%p,%x)\n"
_ALIGN
.text
3:
#endif
cmp %o2, BCOPY_SMALL
Lmemcpy_start:
bge,pt CCCR, 2f ! if >= this many, go be fancy.
cmp %o2, 256
mov %o1, %o5 ! Save memcpy return value
/*
* Not much to copy, just do it a byte at a time.
*/
deccc %o2 ! while (--len >= 0)
bl 1f
.empty
0:
inc %o0
ldsb [%o0 - 1], %o4 ! (++dst)[-1] = *src++;
stb %o4, [%o1]
deccc %o2
bge 0b
inc %o1
1:
retl
mov %o5, %o0
NOTREACHED
/*
* Plenty of data to copy, so try to do it optimally.
*/
2:
#ifdef USE_BLOCK_STORE_LOAD
! If it is big enough, use VIS instructions
bge Lmemcpy_block
nop
#endif /* USE_BLOCK_STORE_LOAD */
Lmemcpy_fancy:
!!
!! First align the output to a 8-byte entity
!!
save %sp, -CC64FSZ, %sp
mov %i0, %l0
mov %i1, %l1
mov %i2, %l2
btst 1, %l1
bz,pt %icc, 4f
btst 2, %l1
ldub [%l0], %l4 ! Load 1st byte
deccc 1, %l2
ble,pn CCCR, Lmemcpy_finish ! XXXX
inc 1, %l0
stb %l4, [%l1] ! Store 1st byte
inc 1, %l1 ! Update address
btst 2, %l1
4:
bz,pt %icc, 4f
btst 1, %l0
bz,a 1f
lduh [%l0], %l4 ! Load short
ldub [%l0], %l4 ! Load bytes
ldub [%l0+1], %l3
sllx %l4, 8, %l4
or %l3, %l4, %l4
1:
deccc 2, %l2
ble,pn CCCR, Lmemcpy_finish ! XXXX
inc 2, %l0
sth %l4, [%l1] ! Store 1st short
inc 2, %l1
4:
btst 4, %l1
bz,pt CCCR, 4f
btst 3, %l0
bz,a,pt CCCR, 1f
lduw [%l0], %l4 ! Load word -1
btst 1, %l0
bz,a,pt %icc, 2f
lduh [%l0], %l4
ldub [%l0], %l4
lduh [%l0+1], %l3
sllx %l4, 16, %l4
or %l4, %l3, %l4
ldub [%l0+3], %l3
sllx %l4, 8, %l4
ba,pt %icc, 1f
or %l4, %l3, %l4
2:
lduh [%l0+2], %l3
sllx %l4, 16, %l4
or %l4, %l3, %l4
1:
deccc 4, %l2
ble,pn CCCR, Lmemcpy_finish ! XXXX
inc 4, %l0
st %l4, [%l1] ! Store word
inc 4, %l1
4:
!!
!! We are now 32-bit aligned in the dest.
!!
Lmemcpy_common:
and %l0, 7, %l4 ! Shift amount
andn %l0, 7, %l0 ! Source addr
brz,pt %l4, Lmemcpy_noshift8 ! No shift version...
sllx %l4, 3, %l4 ! In bits
mov 8<<3, %l3
ldx [%l0], %o0 ! Load word -1
sub %l3, %l4, %l3 ! Reverse shift
deccc 12*8, %l2 ! Have enough room?
sllx %o0, %l4, %o0
bl,pn CCCR, 2f
and %l3, 0x38, %l3
Lmemcpy_unrolled8:
/*
* This is about as close to optimal as you can get, since
* the shifts require EU0 and cannot be paired, and you have
* 3 dependent operations on the data.
*/
! ldx [%l0+0*8], %o0 ! Already done
! sllx %o0, %l4, %o0 ! Already done
ldx [%l0+1*8], %o1
ldx [%l0+2*8], %o2
ldx [%l0+3*8], %o3
ldx [%l0+4*8], %o4
ba,pt %icc, 1f
ldx [%l0+5*8], %o5
.align 8
1:
srlx %o1, %l3, %g1
inc 6*8, %l0
sllx %o1, %l4, %o1
or %g1, %o0, %g6
ldx [%l0+0*8], %o0
stx %g6, [%l1+0*8]
srlx %o2, %l3, %g1
sllx %o2, %l4, %o2
or %g1, %o1, %g6
ldx [%l0+1*8], %o1
stx %g6, [%l1+1*8]
srlx %o3, %l3, %g1
sllx %o3, %l4, %o3
or %g1, %o2, %g6
ldx [%l0+2*8], %o2
stx %g6, [%l1+2*8]
srlx %o4, %l3, %g1
sllx %o4, %l4, %o4
or %g1, %o3, %g6
ldx [%l0+3*8], %o3
stx %g6, [%l1+3*8]
srlx %o5, %l3, %g1
sllx %o5, %l4, %o5
or %g1, %o4, %g6
ldx [%l0+4*8], %o4
stx %g6, [%l1+4*8]
srlx %o0, %l3, %g1
deccc 6*8, %l2 ! Have enough room?
sllx %o0, %l4, %o0 ! Next loop
or %g1, %o5, %g6
ldx [%l0+5*8], %o5
stx %g6, [%l1+5*8]
bge,pt CCCR, 1b
inc 6*8, %l1
Lmemcpy_unrolled8_cleanup:
!!
!! Finished 8 byte block, unload the regs.
!!
srlx %o1, %l3, %g1
inc 5*8, %l0
sllx %o1, %l4, %o1
or %g1, %o0, %g6
stx %g6, [%l1+0*8]
srlx %o2, %l3, %g1
sllx %o2, %l4, %o2
or %g1, %o1, %g6
stx %g6, [%l1+1*8]
srlx %o3, %l3, %g1
sllx %o3, %l4, %o3
or %g1, %o2, %g6
stx %g6, [%l1+2*8]
srlx %o4, %l3, %g1
sllx %o4, %l4, %o4
or %g1, %o3, %g6
stx %g6, [%l1+3*8]
srlx %o5, %l3, %g1
sllx %o5, %l4, %o5
or %g1, %o4, %g6
stx %g6, [%l1+4*8]
inc 5*8, %l1
mov %o5, %o0 ! Save our unused data
dec 5*8, %l2
2:
inccc 12*8, %l2
bz,pn %icc, Lmemcpy_complete
!! Unrolled 8 times
Lmemcpy_aligned8:
! ldx [%l0], %o0 ! Already done
! sllx %o0, %l4, %o0 ! Shift high word
deccc 8, %l2 ! Pre-decrement
bl,pn CCCR, Lmemcpy_finish
1:
ldx [%l0+8], %o1 ! Load word 0
inc 8, %l0
srlx %o1, %l3, %g6
or %g6, %o0, %g6 ! Combine
stx %g6, [%l1] ! Store result
inc 8, %l1
deccc 8, %l2
bge,pn CCCR, 1b
sllx %o1, %l4, %o0
btst 7, %l2 ! Done?
bz,pt CCCR, Lmemcpy_complete
!!
!! Loadup the last dregs into %o0 and shift it into place
!!
srlx %l3, 3, %g6 ! # bytes in %o0
dec 8, %g6 ! - 8
!! n-8 - (by - 8) -> n - by
subcc %l2, %g6, %g0 ! # bytes we need
ble,pt %icc, Lmemcpy_finish
nop
ldx [%l0+8], %o1 ! Need another word
srlx %o1, %l3, %o1
ba,pt %icc, Lmemcpy_finish
or %o0, %o1, %o0 ! All loaded up.
Lmemcpy_noshift8:
deccc 6*8, %l2 ! Have enough room?
bl,pn CCCR, 2f
nop
ba,pt %icc, 1f
nop
.align 32
1:
ldx [%l0+0*8], %o0
ldx [%l0+1*8], %o1
ldx [%l0+2*8], %o2
stx %o0, [%l1+0*8]
stx %o1, [%l1+1*8]
stx %o2, [%l1+2*8]
ldx [%l0+3*8], %o3
ldx [%l0+4*8], %o4
ldx [%l0+5*8], %o5
inc 6*8, %l0
stx %o3, [%l1+3*8]
deccc 6*8, %l2
stx %o4, [%l1+4*8]
stx %o5, [%l1+5*8]
bge,pt CCCR, 1b
inc 6*8, %l1
2:
inc 6*8, %l2
1:
deccc 8, %l2
bl,pn %icc, 1f ! < 0 --> sub word
nop
ldx [%l0], %g6
inc 8, %l0
stx %g6, [%l1]
bg,pt %icc, 1b ! Exactly 0 --> done
inc 8, %l1
1:
btst 7, %l2 ! Done?
bz,pt CCCR, Lmemcpy_complete
clr %l4
ldx [%l0], %o0
Lmemcpy_finish:
brz,pn %l2, 2f ! 100% complete?
cmp %l2, 8 ! Exactly 8 bytes?
bz,a,pn CCCR, 2f
stx %o0, [%l1]
btst 4, %l2 ! Word store?
bz CCCR, 1f
srlx %o0, 32, %g6 ! Shift high word down
stw %g6, [%l1]
inc 4, %l1
mov %o0, %g6 ! Operate on the low bits
1:
btst 2, %l2
mov %g6, %o0
bz 1f
srlx %o0, 16, %g6
sth %g6, [%l1] ! Store short
inc 2, %l1
mov %o0, %g6 ! Operate on low bytes
1:
mov %g6, %o0
btst 1, %l2 ! Byte aligned?
bz 2f
srlx %o0, 8, %g6
stb %g6, [%l1] ! Store last byte
inc 1, %l1 ! Update address
2:
Lmemcpy_complete:
#if 0
!!
!! verify copy success.
!!
mov %i0, %o2
mov %i1, %o4
mov %i2, %l4
0:
ldub [%o2], %o1
inc %o2
ldub [%o4], %o3
inc %o4
cmp %o3, %o1
bnz 1f
dec %l4
brnz %l4, 0b
nop
ba 2f
nop
1:
set 0f, %o0
call printf
sub %i2, %l4, %o5
set 1f, %o0
mov %i0, %o2
mov %i1, %o1
call printf
mov %i2, %o3
ta 1
.data
0: .asciz "memcpy failed: %x@%p != %x@%p byte %d\n"
1: .asciz "memcpy(%p, %p, %lx)\n"
.align 8
.text
2:
#endif
ret
restore %i1, %g0, %o0
#ifdef USE_BLOCK_STORE_LOAD
/*
* Block copy. Useful for >256 byte copies.
*
* Benchmarking has shown this always seems to be slower than
* the integer version, so this is disabled. Maybe someone will
* figure out why sometime.
*/
Lmemcpy_block:
sethi %hi(block_disable), %o3
ldx [ %o3 + %lo(block_disable) ], %o3
brnz,pn %o3, Lmemcpy_fancy
!! Make sure our trap table is installed
set _C_LABEL(trapbase), %o5
rdpr %tba, %o3
sub %o3, %o5, %o3
brnz,pn %o3, Lmemcpy_fancy ! No, then don't use block load/store
nop
#if defined(_KERNEL) && !defined(_RUMPKERNEL)
/*
* Kernel:
*
* Here we use VIS instructions to do a block clear of a page.
* But before we can do that we need to save and enable the FPU.
* The last owner of the FPU registers is fplwp, and
* fplwp->l_md.md_fpstate is the current fpstate. If that's not
* null, call savefpstate() with it to store our current fp state.
*
* Next, allocate an aligned fpstate on the stack. We will properly
* nest calls on a particular stack so this should not be a problem.
*
* Now we grab either curlwp (or if we're on the interrupt stack
* lwp0). We stash its existing fpstate in a local register and
* put our new fpstate in curlwp->p_md.md_fpstate. We point
* fplwp at curlwp (or lwp0) and enable the FPU.
*
* If we are ever preempted, our FPU state will be saved in our
* fpstate. Then, when we're resumed and we take an FPDISABLED
* trap, the trap handler will be able to fish our FPU state out
* of curlwp (or lwp0).
*
* On exiting this routine we undo the damage: restore the original
* pointer to curlwp->p_md.md_fpstate, clear our fplwp, and disable
* the MMU.
*
*
* Register usage, Kernel only (after save):
*
* %i0 src
* %i1 dest
* %i2 size
*
* %l0 XXXX DEBUG old fpstate
* %l1 fplwp (hi bits only)
* %l2 orig fplwp
* %l3 orig fpstate
* %l5 curlwp
* %l6 old fpstate
*
* Register ussage, Kernel and user:
*
* %g1 src (retval for memcpy)
*
* %o0 src
* %o1 dest
* %o2 end dest
* %o5 last safe fetchable address
*/
ENABLE_FPU(0)
mov %i0, %o0 ! Src addr.
mov %i1, %o1 ! Store our dest ptr here.
mov %i2, %o2 ! Len counter
#endif /* _KERNEL */
!!
!! First align the output to a 64-bit entity
!!
mov %o1, %g1 ! memcpy retval
add %o0, %o2, %o5 ! End of source block
andn %o0, 7, %o3 ! Start of block
dec %o5
fzero %f0
andn %o5, BLOCK_ALIGN, %o5 ! Last safe addr.
ldd [%o3], %f2 ! Load 1st word
dec 8, %o3 ! Move %o3 1 word back
btst 1, %o1
bz 4f
mov -7, %o4 ! Lowest src addr possible
alignaddr %o0, %o4, %o4 ! Base addr for load.
cmp %o3, %o4
be,pt CCCR, 1f ! Already loaded?
mov %o4, %o3
fmovd %f2, %f0 ! No. Shift
ldd [%o3+8], %f2 ! And load
1:
faligndata %f0, %f2, %f4 ! Isolate 1st byte
stda %f4, [%o1] ASI_FL8_P ! Store 1st byte
inc 1, %o1 ! Update address
inc 1, %o0
dec 1, %o2
4:
btst 2, %o1
bz 4f
mov -6, %o4 ! Calculate src - 6
alignaddr %o0, %o4, %o4 ! calculate shift mask and dest.
cmp %o3, %o4 ! Addresses same?
be,pt CCCR, 1f
mov %o4, %o3
fmovd %f2, %f0 ! Shuffle data
ldd [%o3+8], %f2 ! Load word 0
1:
faligndata %f0, %f2, %f4 ! Move 1st short low part of f8
stda %f4, [%o1] ASI_FL16_P ! Store 1st short
dec 2, %o2
inc 2, %o1
inc 2, %o0
4:
brz,pn %o2, Lmemcpy_blockfinish ! XXXX
btst 4, %o1
bz 4f
mov -4, %o4
alignaddr %o0, %o4, %o4 ! calculate shift mask and dest.
cmp %o3, %o4 ! Addresses same?
beq,pt CCCR, 1f
mov %o4, %o3
fmovd %f2, %f0 ! Shuffle data
ldd [%o3+8], %f2 ! Load word 0
1:
faligndata %f0, %f2, %f4 ! Move 1st short low part of f8
st %f5, [%o1] ! Store word
dec 4, %o2
inc 4, %o1
inc 4, %o0
4:
brz,pn %o2, Lmemcpy_blockfinish ! XXXX
!!
!! We are now 32-bit aligned in the dest.
!!
Lmemcpy_block_common:
mov -0, %o4
alignaddr %o0, %o4, %o4 ! base - shift
cmp %o3, %o4 ! Addresses same?
beq,pt CCCR, 1f
mov %o4, %o3
fmovd %f2, %f0 ! Shuffle data
ldd [%o3+8], %f2 ! Load word 0
1:
add %o3, 8, %o0 ! now use %o0 for src
!!
!! Continue until our dest is block aligned
!!
Lmemcpy_block_aligned8:
1:
brz %o2, Lmemcpy_blockfinish
btst BLOCK_ALIGN, %o1 ! Block aligned?
bz 1f
faligndata %f0, %f2, %f4 ! Generate result
deccc 8, %o2
ble,pn %icc, Lmemcpy_blockfinish ! Should never happen
fmovd %f4, %f48
std %f4, [%o1] ! Store result
inc 8, %o1
fmovd %f2, %f0
inc 8, %o0
ba,pt %xcc, 1b ! Not yet.
ldd [%o0], %f2 ! Load next part
Lmemcpy_block_aligned64:
1:
/*
* 64-byte aligned -- ready for block operations.
*
* Here we have the destination block aligned, but the
* source pointer may not be. Sub-word alignment will
* be handled by faligndata instructions. But the source
* can still be potentially aligned to 8 different words
* in our 64-bit block, so we have 8 different copy routines.
*
* Once we figure out our source alignment, we branch
* to the appropriate copy routine, which sets up the
* alignment for faligndata and loads (sets) the values
* into the source registers and does the copy loop.
*
* When were down to less than 1 block to store, we
* exit the copy loop and execute cleanup code.
*
* Block loads and stores are not properly interlocked.
* Stores save one reg/cycle, so you can start overwriting
* registers the cycle after the store is issued.
*
* Block loads require a block load to a different register
* block or a membar #Sync before accessing the loaded
* data.
*
* Since the faligndata instructions may be offset as far
* as 7 registers into a block (if you are shifting source
* 7 -> dest 0), you need 3 source register blocks for full
* performance: one you are copying, one you are loading,
* and one for interlocking. Otherwise, we would need to
* sprinkle the code with membar #Sync and lose the advantage
* of running faligndata in parallel with block stores. This
* means we are fetching a full 128 bytes ahead of the stores.
* We need to make sure the prefetch does not inadvertently
* cross a page boundary and fault on data that we will never
* store.
*
*/
#if 1
and %o0, BLOCK_ALIGN, %o3
srax %o3, 3, %o3 ! Isolate the offset
brz %o3, L100 ! 0->0
btst 4, %o3
bnz %xcc, 4f
btst 2, %o3
bnz %xcc, 2f
btst 1, %o3
ba,pt %xcc, L101 ! 0->1
nop /* XXX spitfire bug */
2:
bz %xcc, L102 ! 0->2
nop
ba,pt %xcc, L103 ! 0->3
nop /* XXX spitfire bug */
4:
bnz %xcc, 2f
btst 1, %o3
bz %xcc, L104 ! 0->4
nop
ba,pt %xcc, L105 ! 0->5
nop /* XXX spitfire bug */
2:
bz %xcc, L106 ! 0->6
nop
ba,pt %xcc, L107 ! 0->7
nop /* XXX spitfire bug */
#else
!!
!! Isolate the word offset, which just happens to be
!! the slot in our jump table.
!!
!! This is 6 insns, most of which cannot be paired,
!! which is about the same as the above version.
!!
rd %pc, %o4
1:
and %o0, 0x31, %o3
add %o3, (Lmemcpy_block_jmp - 1b), %o3
jmpl %o4 + %o3, %g0
nop
!!
!! Jump table
!!
Lmemcpy_block_jmp:
ba,a,pt %xcc, L100
nop
ba,a,pt %xcc, L101
nop
ba,a,pt %xcc, L102
nop
ba,a,pt %xcc, L103
nop
ba,a,pt %xcc, L104
nop
ba,a,pt %xcc, L105
nop
ba,a,pt %xcc, L106
nop
ba,a,pt %xcc, L107
nop
#endif
!!
!! Source is block aligned.
!!
!! Just load a block and go.
!!
L100:
#ifdef RETURN_NAME
sethi %hi(1f), %g1
ba,pt %icc, 2f
or %g1, %lo(1f), %g1
1:
.asciz "L100"
.align 8
2:
#endif
fmovd %f0 , %f62
ldda [%o0] ASI_BLK_P, %f0
inc BLOCK_SIZE, %o0
cmp %o0, %o5
bleu,a,pn %icc, 3f
ldda [%o0] ASI_BLK_P, %f16
ba,pt %icc, 3f
membar #Sync
.align 32 ! ICache align.
3:
faligndata %f62, %f0, %f32
inc BLOCK_SIZE, %o0
faligndata %f0, %f2, %f34
dec BLOCK_SIZE, %o2
faligndata %f2, %f4, %f36
cmp %o0, %o5
faligndata %f4, %f6, %f38
faligndata %f6, %f8, %f40
faligndata %f8, %f10, %f42
faligndata %f10, %f12, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f12, %f14, %f46
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f48
membar #Sync
2:
stda %f32, [%o1] ASI_STORE
faligndata %f14, %f16, %f32
inc BLOCK_SIZE, %o0
faligndata %f16, %f18, %f34
inc BLOCK_SIZE, %o1
faligndata %f18, %f20, %f36
dec BLOCK_SIZE, %o2
faligndata %f20, %f22, %f38
cmp %o0, %o5
faligndata %f22, %f24, %f40
faligndata %f24, %f26, %f42
faligndata %f26, %f28, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f28, %f30, %f46
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f0
membar #Sync
2:
stda %f32, [%o1] ASI_STORE
faligndata %f30, %f48, %f32
inc BLOCK_SIZE, %o0
faligndata %f48, %f50, %f34
inc BLOCK_SIZE, %o1
faligndata %f50, %f52, %f36
dec BLOCK_SIZE, %o2
faligndata %f52, %f54, %f38
cmp %o0, %o5
faligndata %f54, %f56, %f40
faligndata %f56, %f58, %f42
faligndata %f58, %f60, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f60, %f62, %f46
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f16 ! Increment is at top
membar #Sync
2:
stda %f32, [%o1] ASI_STORE
ba 3b
inc BLOCK_SIZE, %o1
!!
!! Source at BLOCK_ALIGN+8
!!
!! We need to load almost 1 complete block by hand.
!!
L101:
#ifdef RETURN_NAME
sethi %hi(1f), %g1
ba,pt %icc, 2f
or %g1, %lo(1f), %g1
1:
.asciz "L101"
.align 8
2:
#endif
! fmovd %f0, %f0 ! Hoist fmovd
ldd [%o0], %f2
inc 8, %o0
ldd [%o0], %f4
inc 8, %o0
ldd [%o0], %f6
inc 8, %o0
ldd [%o0], %f8
inc 8, %o0
ldd [%o0], %f10
inc 8, %o0
ldd [%o0], %f12
inc 8, %o0
ldd [%o0], %f14
inc 8, %o0
cmp %o0, %o5
bleu,a,pn %icc, 3f
ldda [%o0] ASI_BLK_P, %f16
membar #Sync
3:
faligndata %f0, %f2, %f32
inc BLOCK_SIZE, %o0
faligndata %f2, %f4, %f34
cmp %o0, %o5
faligndata %f4, %f6, %f36
dec BLOCK_SIZE, %o2
faligndata %f6, %f8, %f38
faligndata %f8, %f10, %f40
faligndata %f10, %f12, %f42
faligndata %f12, %f14, %f44
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f48
membar #Sync
2:
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f14, %f16, %f46
stda %f32, [%o1] ASI_STORE
faligndata %f16, %f18, %f32
inc BLOCK_SIZE, %o0
faligndata %f18, %f20, %f34
inc BLOCK_SIZE, %o1
faligndata %f20, %f22, %f36
cmp %o0, %o5
faligndata %f22, %f24, %f38
dec BLOCK_SIZE, %o2
faligndata %f24, %f26, %f40
faligndata %f26, %f28, %f42
faligndata %f28, %f30, %f44
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f0
membar #Sync
2:
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f30, %f48, %f46
stda %f32, [%o1] ASI_STORE
faligndata %f48, %f50, %f32
inc BLOCK_SIZE, %o0
faligndata %f50, %f52, %f34
inc BLOCK_SIZE, %o1
faligndata %f52, %f54, %f36
cmp %o0, %o5
faligndata %f54, %f56, %f38
dec BLOCK_SIZE, %o2
faligndata %f56, %f58, %f40
faligndata %f58, %f60, %f42
faligndata %f60, %f62, %f44
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f16
membar #Sync
2:
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f62, %f0, %f46
stda %f32, [%o1] ASI_STORE
ba 3b
inc BLOCK_SIZE, %o1
!!
!! Source at BLOCK_ALIGN+16
!!
!! We need to load 6 doubles by hand.
!!
L102:
#ifdef RETURN_NAME
sethi %hi(1f), %g1
ba,pt %icc, 2f
or %g1, %lo(1f), %g1
1:
.asciz "L102"
.align 8
2:
#endif
ldd [%o0], %f4
inc 8, %o0
fmovd %f0, %f2 ! Hoist fmovd
ldd [%o0], %f6
inc 8, %o0
ldd [%o0], %f8
inc 8, %o0
ldd [%o0], %f10
inc 8, %o0
ldd [%o0], %f12
inc 8, %o0
ldd [%o0], %f14
inc 8, %o0
cmp %o0, %o5
bleu,a,pn %icc, 3f
ldda [%o0] ASI_BLK_P, %f16
membar #Sync
3:
faligndata %f2, %f4, %f32
inc BLOCK_SIZE, %o0
faligndata %f4, %f6, %f34
cmp %o0, %o5
faligndata %f6, %f8, %f36
dec BLOCK_SIZE, %o2
faligndata %f8, %f10, %f38
faligndata %f10, %f12, %f40
faligndata %f12, %f14, %f42
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f48
membar #Sync
2:
faligndata %f14, %f16, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f16, %f18, %f46
stda %f32, [%o1] ASI_STORE
faligndata %f18, %f20, %f32
inc BLOCK_SIZE, %o0
faligndata %f20, %f22, %f34
inc BLOCK_SIZE, %o1
faligndata %f22, %f24, %f36
cmp %o0, %o5
faligndata %f24, %f26, %f38
dec BLOCK_SIZE, %o2
faligndata %f26, %f28, %f40
faligndata %f28, %f30, %f42
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f0
membar #Sync
2:
faligndata %f30, %f48, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f48, %f50, %f46
stda %f32, [%o1] ASI_STORE
faligndata %f50, %f52, %f32
inc BLOCK_SIZE, %o0
faligndata %f52, %f54, %f34
inc BLOCK_SIZE, %o1
faligndata %f54, %f56, %f36
cmp %o0, %o5
faligndata %f56, %f58, %f38
dec BLOCK_SIZE, %o2
faligndata %f58, %f60, %f40
faligndata %f60, %f62, %f42
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f16
membar #Sync
2:
faligndata %f62, %f0, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f0, %f2, %f46
stda %f32, [%o1] ASI_STORE
ba 3b
inc BLOCK_SIZE, %o1
!!
!! Source at BLOCK_ALIGN+24
!!
!! We need to load 5 doubles by hand.
!!
L103:
#ifdef RETURN_NAME
sethi %hi(1f), %g1
ba,pt %icc, 2f
or %g1, %lo(1f), %g1
1:
.asciz "L103"
.align 8
2:
#endif
fmovd %f0, %f4
ldd [%o0], %f6
inc 8, %o0
ldd [%o0], %f8
inc 8, %o0
ldd [%o0], %f10
inc 8, %o0
ldd [%o0], %f12
inc 8, %o0
ldd [%o0], %f14
inc 8, %o0
cmp %o0, %o5
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f16
membar #Sync
2:
inc BLOCK_SIZE, %o0
3:
faligndata %f4, %f6, %f32
cmp %o0, %o5
faligndata %f6, %f8, %f34
dec BLOCK_SIZE, %o2
faligndata %f8, %f10, %f36
faligndata %f10, %f12, %f38
faligndata %f12, %f14, %f40
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f48
membar #Sync
2:
faligndata %f14, %f16, %f42
inc BLOCK_SIZE, %o0
faligndata %f16, %f18, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f18, %f20, %f46
stda %f32, [%o1] ASI_STORE
faligndata %f20, %f22, %f32
cmp %o0, %o5
faligndata %f22, %f24, %f34
dec BLOCK_SIZE, %o2
faligndata %f24, %f26, %f36
inc BLOCK_SIZE, %o1
faligndata %f26, %f28, %f38
faligndata %f28, %f30, %f40
ble,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f0
membar #Sync
2:
faligndata %f30, %f48, %f42
inc BLOCK_SIZE, %o0
faligndata %f48, %f50, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f50, %f52, %f46
stda %f32, [%o1] ASI_STORE
faligndata %f52, %f54, %f32
cmp %o0, %o5
faligndata %f54, %f56, %f34
dec BLOCK_SIZE, %o2
faligndata %f56, %f58, %f36
faligndata %f58, %f60, %f38
inc BLOCK_SIZE, %o1
faligndata %f60, %f62, %f40
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f16
membar #Sync
2:
faligndata %f62, %f0, %f42
inc BLOCK_SIZE, %o0
faligndata %f0, %f2, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f2, %f4, %f46
stda %f32, [%o1] ASI_STORE
ba 3b
inc BLOCK_SIZE, %o1
!!
!! Source at BLOCK_ALIGN+32
!!
!! We need to load 4 doubles by hand.
!!
L104:
#ifdef RETURN_NAME
sethi %hi(1f), %g1
ba,pt %icc, 2f
or %g1, %lo(1f), %g1
1:
.asciz "L104"
.align 8
2:
#endif
fmovd %f0, %f6
ldd [%o0], %f8
inc 8, %o0
ldd [%o0], %f10
inc 8, %o0
ldd [%o0], %f12
inc 8, %o0
ldd [%o0], %f14
inc 8, %o0
cmp %o0, %o5
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f16
membar #Sync
2:
inc BLOCK_SIZE, %o0
3:
faligndata %f6, %f8, %f32
cmp %o0, %o5
faligndata %f8, %f10, %f34
dec BLOCK_SIZE, %o2
faligndata %f10, %f12, %f36
faligndata %f12, %f14, %f38
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f48
membar #Sync
2:
faligndata %f14, %f16, %f40
faligndata %f16, %f18, %f42
inc BLOCK_SIZE, %o0
faligndata %f18, %f20, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f20, %f22, %f46
stda %f32, [%o1] ASI_STORE
faligndata %f22, %f24, %f32
cmp %o0, %o5
faligndata %f24, %f26, %f34
faligndata %f26, %f28, %f36
inc BLOCK_SIZE, %o1
faligndata %f28, %f30, %f38
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f0
membar #Sync
2:
faligndata %f30, %f48, %f40
dec BLOCK_SIZE, %o2
faligndata %f48, %f50, %f42
inc BLOCK_SIZE, %o0
faligndata %f50, %f52, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f52, %f54, %f46
stda %f32, [%o1] ASI_STORE
faligndata %f54, %f56, %f32
cmp %o0, %o5
faligndata %f56, %f58, %f34
faligndata %f58, %f60, %f36
inc BLOCK_SIZE, %o1
faligndata %f60, %f62, %f38
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f16
membar #Sync
2:
faligndata %f62, %f0, %f40
dec BLOCK_SIZE, %o2
faligndata %f0, %f2, %f42
inc BLOCK_SIZE, %o0
faligndata %f2, %f4, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f4, %f6, %f46
stda %f32, [%o1] ASI_STORE
ba 3b
inc BLOCK_SIZE, %o1
!!
!! Source at BLOCK_ALIGN+40
!!
!! We need to load 3 doubles by hand.
!!
L105:
#ifdef RETURN_NAME
sethi %hi(1f), %g1
ba,pt %icc, 2f
or %g1, %lo(1f), %g1
1:
.asciz "L105"
.align 8
2:
#endif
fmovd %f0, %f8
ldd [%o0], %f10
inc 8, %o0
ldd [%o0], %f12
inc 8, %o0
ldd [%o0], %f14
inc 8, %o0
cmp %o0, %o5
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f16
membar #Sync
2:
inc BLOCK_SIZE, %o0
3:
faligndata %f8, %f10, %f32
cmp %o0, %o5
faligndata %f10, %f12, %f34
faligndata %f12, %f14, %f36
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f48
membar #Sync
2:
faligndata %f14, %f16, %f38
dec BLOCK_SIZE, %o2
faligndata %f16, %f18, %f40
inc BLOCK_SIZE, %o0
faligndata %f18, %f20, %f42
faligndata %f20, %f22, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f22, %f24, %f46
stda %f32, [%o1] ASI_STORE
faligndata %f24, %f26, %f32
cmp %o0, %o5
faligndata %f26, %f28, %f34
dec BLOCK_SIZE, %o2
faligndata %f28, %f30, %f36
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f0
membar #Sync
2:
faligndata %f30, %f48, %f38
inc BLOCK_SIZE, %o1
faligndata %f48, %f50, %f40
inc BLOCK_SIZE, %o0
faligndata %f50, %f52, %f42
faligndata %f52, %f54, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f54, %f56, %f46
stda %f32, [%o1] ASI_STORE
faligndata %f56, %f58, %f32
cmp %o0, %o5
faligndata %f58, %f60, %f34
dec BLOCK_SIZE, %o2
faligndata %f60, %f62, %f36
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f16
membar #Sync
2:
faligndata %f62, %f0, %f38
inc BLOCK_SIZE, %o1
faligndata %f0, %f2, %f40
inc BLOCK_SIZE, %o0
faligndata %f2, %f4, %f42
faligndata %f4, %f6, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f6, %f8, %f46
stda %f32, [%o1] ASI_STORE
ba 3b
inc BLOCK_SIZE, %o1
!!
!! Source at BLOCK_ALIGN+48
!!
!! We need to load 2 doubles by hand.
!!
L106:
#ifdef RETURN_NAME
sethi %hi(1f), %g1
ba,pt %icc, 2f
or %g1, %lo(1f), %g1
1:
.asciz "L106"
.align 8
2:
#endif
fmovd %f0, %f10
ldd [%o0], %f12
inc 8, %o0
ldd [%o0], %f14
inc 8, %o0
cmp %o0, %o5
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f16
membar #Sync
2:
inc BLOCK_SIZE, %o0
3:
faligndata %f10, %f12, %f32
cmp %o0, %o5
faligndata %f12, %f14, %f34
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f48
membar #Sync
2:
faligndata %f14, %f16, %f36
dec BLOCK_SIZE, %o2
faligndata %f16, %f18, %f38
inc BLOCK_SIZE, %o0
faligndata %f18, %f20, %f40
faligndata %f20, %f22, %f42
faligndata %f22, %f24, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f24, %f26, %f46
stda %f32, [%o1] ASI_STORE
faligndata %f26, %f28, %f32
cmp %o0, %o5
faligndata %f28, %f30, %f34
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f0
membar #Sync
2:
faligndata %f30, %f48, %f36
dec BLOCK_SIZE, %o2
faligndata %f48, %f50, %f38
inc BLOCK_SIZE, %o1
faligndata %f50, %f52, %f40
faligndata %f52, %f54, %f42
inc BLOCK_SIZE, %o0
faligndata %f54, %f56, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f56, %f58, %f46
stda %f32, [%o1] ASI_STORE
faligndata %f58, %f60, %f32
cmp %o0, %o5
faligndata %f60, %f62, %f34
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f16
membar #Sync
2:
faligndata %f62, %f0, %f36
dec BLOCK_SIZE, %o2
faligndata %f0, %f2, %f38
inc BLOCK_SIZE, %o1
faligndata %f2, %f4, %f40
faligndata %f4, %f6, %f42
inc BLOCK_SIZE, %o0
faligndata %f6, %f8, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f8, %f10, %f46
stda %f32, [%o1] ASI_STORE
ba 3b
inc BLOCK_SIZE, %o1
!!
!! Source at BLOCK_ALIGN+56
!!
!! We need to load 1 double by hand.
!!
L107:
#ifdef RETURN_NAME
sethi %hi(1f), %g1
ba,pt %icc, 2f
or %g1, %lo(1f), %g1
1:
.asciz "L107"
.align 8
2:
#endif
fmovd %f0, %f12
ldd [%o0], %f14
inc 8, %o0
cmp %o0, %o5
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f16
membar #Sync
2:
inc BLOCK_SIZE, %o0
3:
faligndata %f12, %f14, %f32
cmp %o0, %o5
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f48
membar #Sync
2:
faligndata %f14, %f16, %f34
dec BLOCK_SIZE, %o2
faligndata %f16, %f18, %f36
inc BLOCK_SIZE, %o0
faligndata %f18, %f20, %f38
faligndata %f20, %f22, %f40
faligndata %f22, %f24, %f42
faligndata %f24, %f26, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f26, %f28, %f46
stda %f32, [%o1] ASI_STORE
faligndata %f28, %f30, %f32
cmp %o0, %o5
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f0
membar #Sync
2:
faligndata %f30, %f48, %f34
dec BLOCK_SIZE, %o2
faligndata %f48, %f50, %f36
inc BLOCK_SIZE, %o1
faligndata %f50, %f52, %f38
faligndata %f52, %f54, %f40
inc BLOCK_SIZE, %o0
faligndata %f54, %f56, %f42
faligndata %f56, %f58, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f58, %f60, %f46
stda %f32, [%o1] ASI_STORE
faligndata %f60, %f62, %f32
cmp %o0, %o5
bleu,a,pn %icc, 2f
ldda [%o0] ASI_BLK_P, %f16
membar #Sync
2:
faligndata %f62, %f0, %f34
dec BLOCK_SIZE, %o2
faligndata %f0, %f2, %f36
inc BLOCK_SIZE, %o1
faligndata %f2, %f4, %f38
faligndata %f4, %f6, %f40
inc BLOCK_SIZE, %o0
faligndata %f6, %f8, %f42
faligndata %f8, %f10, %f44
brlez,pn %o2, Lmemcpy_blockdone
faligndata %f10, %f12, %f46
stda %f32, [%o1] ASI_STORE
ba 3b
inc BLOCK_SIZE, %o1
Lmemcpy_blockdone:
inc BLOCK_SIZE, %o2 ! Fixup our overcommit
membar #Sync ! Finish any pending loads
#define FINISH_REG(f) \
deccc 8, %o2; \
bl,a Lmemcpy_blockfinish; \
fmovd f, %f48; \
std f, [%o1]; \
inc 8, %o1
FINISH_REG(%f32)
FINISH_REG(%f34)
FINISH_REG(%f36)
FINISH_REG(%f38)
FINISH_REG(%f40)
FINISH_REG(%f42)
FINISH_REG(%f44)
FINISH_REG(%f46)
FINISH_REG(%f48)
#undef FINISH_REG
!!
!! The low 3 bits have the sub-word bits needed to be
!! stored [because (x-8)&0x7 == x].
!!
Lmemcpy_blockfinish:
brz,pn %o2, 2f ! 100% complete?
fmovd %f48, %f4
cmp %o2, 8 ! Exactly 8 bytes?
bz,a,pn CCCR, 2f
std %f4, [%o1]
btst 4, %o2 ! Word store?
bz CCCR, 1f
nop
st %f4, [%o1]
inc 4, %o1
1:
btst 2, %o2
fzero %f0
bz 1f
mov -6, %o4
alignaddr %o1, %o4, %g0
faligndata %f0, %f4, %f8
stda %f8, [%o1] ASI_FL16_P ! Store short
inc 2, %o1
1:
btst 1, %o2 ! Byte aligned?
bz 2f
mov -7, %o0 ! Calculate dest - 7
alignaddr %o1, %o0, %g0 ! Calculate shift mask and dest.
faligndata %f0, %f4, %f8 ! Move 1st byte to low part of f8
stda %f8, [%o1] ASI_FL8_P ! Store 1st byte
inc 1, %o1 ! Update address
2:
membar #Sync
#if 0
!!
!! verify copy success.
!!
mov %i0, %o2
mov %i1, %o4
mov %i2, %l4
0:
ldub [%o2], %o1
inc %o2
ldub [%o4], %o3
inc %o4
cmp %o3, %o1
bnz 1f
dec %l4
brnz %l4, 0b
nop
ba 2f
nop
1:
set block_disable, %o0
stx %o0, [%o0]
set 0f, %o0
call prom_printf
sub %i2, %l4, %o5
set 1f, %o0
mov %i0, %o2
mov %i1, %o1
call prom_printf
mov %i2, %o3
ta 1
.data
_ALIGN
0: .asciz "block memcpy failed: %x@%p != %x@%p byte %d\r\n"
1: .asciz "memcpy(%p, %p, %lx)\r\n"
_ALIGN
.text
2:
#endif
#if defined(_KERNEL) && !defined(_RUMPKERNEL)
/*
* Weve saved our possible fpstate, now disable the fpu
* and continue with life.
*/
RESTORE_FPU
ret
restore %g1, 0, %o0 ! Return DEST for memcpy
#endif
retl
mov %g1, %o0
/*
* Use block_disable to turn off block insns for
* memcpy/memset
*/
.data
.align 8
.globl block_disable
block_disable: .xword 1
.text
#endif /* USE_BLOCK_STORE_LOAD */