Subversion Repositories or1k

/*-

 * Copyright (c) 1997 The NetBSD Foundation, Inc.

 * All rights reserved.

 *

 * This code is derived from software contributed to The NetBSD Foundation

 * by Neil A. Carson and Mark Brinicombe

 *

 * Redistribution and use in source and binary forms, with or without

 * modification, are permitted provided that the following conditions

 * are met:

 * 1. Redistributions of source code must retain the above copyright

 *    notice, this list of conditions and the following disclaimer.

 * 2. Redistributions in binary form must reproduce the above copyright

 *    notice, this list of conditions and the following disclaimer in the

 *    documentation and/or other materials provided with the distribution.

 * 3. All advertising materials mentioning features or use of this software

 *    must display the following acknowledgement:

 *        This product includes software developed by the NetBSD

 *        Foundation, Inc. and its contributors.

 * 4. Neither the name of The NetBSD Foundation nor the names of its

 *    contributors may be used to endorse or promote products derived

 *    from this software without specific prior written permission.

 *

 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS

 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED

 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS

 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR

 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF

 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS

 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN

 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)

 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

 * POSSIBILITY OF SUCH DAMAGE.

 *

 * Adapted for uClibc from NetBSD _memcpy.S,v 1.6 2003/10/09

 * by Erik Andersen 

 */

#include 

/*

 * This is one fun bit of code ...

 * Some easy listening music is suggested while trying to understand this

 * code e.g. Iron Maiden

 *

 * For anyone attempting to understand it :

 *

 * The core code is implemented here with simple stubs for memcpy()

 * memmove() and bcopy().

 *

 * All local labels are prefixed with Lmemcpy_

 * Following the prefix a label starting f is used in the forward copy code

 * while a label using b is used in the backwards copy code

 * The source and destination addresses determine whether a forward or

 * backward copy is performed.

 * Separate bits of code are used to deal with the following situations

 * for both the forward and backwards copy.

 * unaligned source address

 * unaligned destination address

 * Separate copy routines are used to produce an optimised result for each

 * of these cases.

 * The copy code will use LDM/STM instructions to copy up to 32 bytes at

 * a time where possible.

 *

 * Note: r12 (aka ip) can be trashed during the function along with

 * r0-r3 although r0-r2 have defined uses i.e. src, dest, len through out.

 * Additional registers are preserved prior to use i.e. r4, r5 & lr

 *

 * Apologies for the state of the comments ;-)

 */

                .text

                .global _memcpy;

                .type _memcpy,%function

                .align 4;                                                               \

_memcpy:

        /* Determine copy direction */

        cmp     r1, r0

        bcc     .Lmemcpy_backwards

        moveq   r0, #0                  /* Quick abort for len=0 */

        moveq   pc, lr

        stmdb   sp!, {r0, lr}           /* memcpy() returns dest addr */

        subs    r2, r2, #4

        blt     .Lmemcpy_fl4            /* less than 4 bytes */

        ands    r12, r0, #3

        bne     .Lmemcpy_fdestul        /* oh unaligned destination addr */

        ands    r12, r1, #3

        bne     .Lmemcpy_fsrcul         /* oh unaligned source addr */

.Lmemcpy_ft8:

        /* We have aligned source and destination */

        subs    r2, r2, #8

        blt     .Lmemcpy_fl12           /* less than 12 bytes (4 from above) */

        subs    r2, r2, #0x14

        blt     .Lmemcpy_fl32           /* less than 32 bytes (12 from above) */

        stmdb   sp!, {r4}               /* borrow r4 */

        /* blat 32 bytes at a time */

        /* XXX for really big copies perhaps we should use more registers */

.Lmemcpy_floop32:

        ldmia   r1!, {r3, r4, r12, lr}

        stmia   r0!, {r3, r4, r12, lr}

        ldmia   r1!, {r3, r4, r12, lr}

        stmia   r0!, {r3, r4, r12, lr}

        subs    r2, r2, #0x20

        bge     .Lmemcpy_floop32

        cmn     r2, #0x10

        ldmgeia r1!, {r3, r4, r12, lr}  /* blat a remaining 16 bytes */

        stmgeia r0!, {r3, r4, r12, lr}

        subge   r2, r2, #0x10

        ldmia   sp!, {r4}               /* return r4 */

.Lmemcpy_fl32:

        adds    r2, r2, #0x14

        /* blat 12 bytes at a time */

.Lmemcpy_floop12:

        ldmgeia r1!, {r3, r12, lr}

        stmgeia r0!, {r3, r12, lr}

        subges  r2, r2, #0x0c

        bge     .Lmemcpy_floop12

.Lmemcpy_fl12:

        adds    r2, r2, #8

        blt     .Lmemcpy_fl4

        subs    r2, r2, #4

        ldrlt   r3, [r1], #4

        strlt   r3, [r0], #4

        ldmgeia r1!, {r3, r12}

        stmgeia r0!, {r3, r12}

        subge   r2, r2, #4

.Lmemcpy_fl4:

        /* less than 4 bytes to go */

        adds    r2, r2, #4

        ldmeqia sp!, {r0, pc}           /* done */

        /* copy the crud byte at a time */

        cmp     r2, #2

        ldrb    r3, [r1], #1

        strb    r3, [r0], #1

        ldrgeb  r3, [r1], #1

        strgeb  r3, [r0], #1

        ldrgtb  r3, [r1], #1

        strgtb  r3, [r0], #1

        ldmia   sp!, {r0, pc}

        /* erg - unaligned destination */

.Lmemcpy_fdestul:

        rsb     r12, r12, #4

        cmp     r12, #2

        /* align destination with byte copies */

        ldrb    r3, [r1], #1

        strb    r3, [r0], #1

        ldrgeb  r3, [r1], #1

        strgeb  r3, [r0], #1

        ldrgtb  r3, [r1], #1

        strgtb  r3, [r0], #1

        subs    r2, r2, r12

        blt     .Lmemcpy_fl4            /* less the 4 bytes */

        ands    r12, r1, #3

        beq     .Lmemcpy_ft8            /* we have an aligned source */

        /* erg - unaligned source */

        /* This is where it gets nasty ... */

.Lmemcpy_fsrcul:

        bic     r1, r1, #3

        ldr     lr, [r1], #4

        cmp     r12, #2

        bgt     .Lmemcpy_fsrcul3

        beq     .Lmemcpy_fsrcul2

        cmp     r2, #0x0c

        blt     .Lmemcpy_fsrcul1loop4

        sub     r2, r2, #0x0c

        stmdb   sp!, {r4, r5}

.Lmemcpy_fsrcul1loop16:

#if __BYTE_ORDER == __BIG_ENDIAN

        mov     r3, lr, lsl #8

        ldmia   r1!, {r4, r5, r12, lr}

        orr     r3, r3, r4, lsr #24

        mov     r4, r4, lsl #8

        orr     r4, r4, r5, lsr #24

        mov     r5, r5, lsl #8

        orr     r5, r5, r12, lsr #24

        mov     r12, r12, lsl #8

        orr     r12, r12, lr, lsr #24

#else

        mov     r3, lr, lsr #8

        ldmia   r1!, {r4, r5, r12, lr}

        orr     r3, r3, r4, lsl #24

        mov     r4, r4, lsr #8

        orr     r4, r4, r5, lsl #24

        mov     r5, r5, lsr #8

        orr     r5, r5, r12, lsl #24

        mov     r12, r12, lsr #8

        orr     r12, r12, lr, lsl #24

#endif

        stmia   r0!, {r3-r5, r12}

        subs    r2, r2, #0x10

        bge     .Lmemcpy_fsrcul1loop16

        ldmia   sp!, {r4, r5}

        adds    r2, r2, #0x0c

        blt     .Lmemcpy_fsrcul1l4

.Lmemcpy_fsrcul1loop4:

#if __BYTE_ORDER == __BIG_ENDIAN

        mov     r12, lr, lsl #8

        ldr     lr, [r1], #4

        orr     r12, r12, lr, lsr #24

#else

        mov     r12, lr, lsr #8

        ldr     lr, [r1], #4

        orr     r12, r12, lr, lsl #24

#endif

        str     r12, [r0], #4

        subs    r2, r2, #4

        bge     .Lmemcpy_fsrcul1loop4

.Lmemcpy_fsrcul1l4:

        sub     r1, r1, #3

        b       .Lmemcpy_fl4

.Lmemcpy_fsrcul2:

        cmp     r2, #0x0c

        blt     .Lmemcpy_fsrcul2loop4

        sub     r2, r2, #0x0c

        stmdb   sp!, {r4, r5}

.Lmemcpy_fsrcul2loop16:

#if __BYTE_ORDER == __BIG_ENDIAN

        mov     r3, lr, lsl #16

        ldmia   r1!, {r4, r5, r12, lr}

        orr     r3, r3, r4, lsr #16

        mov     r4, r4, lsl #16

        orr     r4, r4, r5, lsr #16

        mov     r5, r5, lsl #16

        orr     r5, r5, r12, lsr #16

        mov     r12, r12, lsl #16

        orr     r12, r12, lr, lsr #16

#else

        mov     r3, lr, lsr #16

        ldmia   r1!, {r4, r5, r12, lr}

        orr     r3, r3, r4, lsl #16

        mov     r4, r4, lsr #16

        orr     r4, r4, r5, lsl #16

        mov     r5, r5, lsr #16

        orr     r5, r5, r12, lsl #16

        mov     r12, r12, lsr #16

        orr     r12, r12, lr, lsl #16

#endif

        stmia   r0!, {r3-r5, r12}

        subs    r2, r2, #0x10

        bge     .Lmemcpy_fsrcul2loop16

        ldmia   sp!, {r4, r5}

        adds    r2, r2, #0x0c

        blt     .Lmemcpy_fsrcul2l4

.Lmemcpy_fsrcul2loop4:

#if __BYTE_ORDER == __BIG_ENDIAN

        mov     r12, lr, lsl #16

        ldr     lr, [r1], #4

        orr     r12, r12, lr, lsr #16

#else

        mov     r12, lr, lsr #16

        ldr     lr, [r1], #4

        orr     r12, r12, lr, lsl #16

#endif

        str     r12, [r0], #4

        subs    r2, r2, #4

        bge     .Lmemcpy_fsrcul2loop4

.Lmemcpy_fsrcul2l4:

        sub     r1, r1, #2

        b       .Lmemcpy_fl4

.Lmemcpy_fsrcul3:

        cmp     r2, #0x0c

        blt     .Lmemcpy_fsrcul3loop4

        sub     r2, r2, #0x0c

        stmdb   sp!, {r4, r5}

.Lmemcpy_fsrcul3loop16:

#if __BYTE_ORDER == __BIG_ENDIAN

        mov     r3, lr, lsl #24

        ldmia   r1!, {r4, r5, r12, lr}

        orr     r3, r3, r4, lsr #8

        mov     r4, r4, lsl #24

        orr     r4, r4, r5, lsr #8

        mov     r5, r5, lsl #24

        orr     r5, r5, r12, lsr #8

        mov     r12, r12, lsl #24

        orr     r12, r12, lr, lsr #8

#else

        mov     r3, lr, lsr #24

        ldmia   r1!, {r4, r5, r12, lr}

        orr     r3, r3, r4, lsl #8

        mov     r4, r4, lsr #24

        orr     r4, r4, r5, lsl #8

        mov     r5, r5, lsr #24

        orr     r5, r5, r12, lsl #8

        mov     r12, r12, lsr #24

        orr     r12, r12, lr, lsl #8

#endif

        stmia   r0!, {r3-r5, r12}

        subs    r2, r2, #0x10

        bge     .Lmemcpy_fsrcul3loop16

        ldmia   sp!, {r4, r5}

        adds    r2, r2, #0x0c

        blt     .Lmemcpy_fsrcul3l4

.Lmemcpy_fsrcul3loop4:

#if __BYTE_ORDER == __BIG_ENDIAN

        mov     r12, lr, lsl #24

        ldr     lr, [r1], #4

        orr     r12, r12, lr, lsr #8

#else

        mov     r12, lr, lsr #24

        ldr     lr, [r1], #4

        orr     r12, r12, lr, lsl #8

#endif

        str     r12, [r0], #4

        subs    r2, r2, #4

        bge     .Lmemcpy_fsrcul3loop4

.Lmemcpy_fsrcul3l4:

        sub     r1, r1, #1

        b       .Lmemcpy_fl4

.Lmemcpy_backwards:

        add     r1, r1, r2

        add     r0, r0, r2

        subs    r2, r2, #4

        blt     .Lmemcpy_bl4            /* less than 4 bytes */

        ands    r12, r0, #3

        bne     .Lmemcpy_bdestul        /* oh unaligned destination addr */

        ands    r12, r1, #3

        bne     .Lmemcpy_bsrcul         /* oh unaligned source addr */

.Lmemcpy_bt8:

        /* We have aligned source and destination */

        subs    r2, r2, #8

        blt     .Lmemcpy_bl12           /* less than 12 bytes (4 from above) */

        stmdb   sp!, {r4, lr}

        subs    r2, r2, #0x14           /* less than 32 bytes (12 from above) */

        blt     .Lmemcpy_bl32

        /* blat 32 bytes at a time */

        /* XXX for really big copies perhaps we should use more registers */

.Lmemcpy_bloop32:

        ldmdb   r1!, {r3, r4, r12, lr}

        stmdb   r0!, {r3, r4, r12, lr}

        ldmdb   r1!, {r3, r4, r12, lr}

        stmdb   r0!, {r3, r4, r12, lr}

        subs    r2, r2, #0x20

        bge     .Lmemcpy_bloop32

.Lmemcpy_bl32:

        cmn     r2, #0x10

        ldmgedb r1!, {r3, r4, r12, lr}  /* blat a remaining 16 bytes */

        stmgedb r0!, {r3, r4, r12, lr}

        subge   r2, r2, #0x10

        adds    r2, r2, #0x14

        ldmgedb r1!, {r3, r12, lr}      /* blat a remaining 12 bytes */

        stmgedb r0!, {r3, r12, lr}

        subge   r2, r2, #0x0c

        ldmia   sp!, {r4, lr}

.Lmemcpy_bl12:

        adds    r2, r2, #8

        blt     .Lmemcpy_bl4

        subs    r2, r2, #4

        ldrlt   r3, [r1, #-4]!

        strlt   r3, [r0, #-4]!

        ldmgedb r1!, {r3, r12}

        stmgedb r0!, {r3, r12}

        subge   r2, r2, #4

.Lmemcpy_bl4:

        /* less than 4 bytes to go */

        adds    r2, r2, #4

        moveq   pc, lr                  /* done */

        /* copy the crud byte at a time */

        cmp     r2, #2

        ldrb    r3, [r1, #-1]!

        strb    r3, [r0, #-1]!

        ldrgeb  r3, [r1, #-1]!

        strgeb  r3, [r0, #-1]!

        ldrgtb  r3, [r1, #-1]!

        strgtb  r3, [r0, #-1]!

        mov     pc, lr

        /* erg - unaligned destination */

.Lmemcpy_bdestul:

        cmp     r12, #2

        /* align destination with byte copies */

        ldrb    r3, [r1, #-1]!

        strb    r3, [r0, #-1]!

        ldrgeb  r3, [r1, #-1]!

        strgeb  r3, [r0, #-1]!

        ldrgtb  r3, [r1, #-1]!

        strgtb  r3, [r0, #-1]!

        subs    r2, r2, r12

        blt     .Lmemcpy_bl4            /* less than 4 bytes to go */

        ands    r12, r1, #3

        beq     .Lmemcpy_bt8            /* we have an aligned source */

        /* erg - unaligned source */

        /* This is where it gets nasty ... */

.Lmemcpy_bsrcul:

        bic     r1, r1, #3

        ldr     r3, [r1, #0]

        cmp     r12, #2

        blt     .Lmemcpy_bsrcul1

        beq     .Lmemcpy_bsrcul2

        cmp     r2, #0x0c

        blt     .Lmemcpy_bsrcul3loop4

        sub     r2, r2, #0x0c

        stmdb   sp!, {r4, r5, lr}

.Lmemcpy_bsrcul3loop16:

#if __BYTE_ORDER == __BIG_ENDIAN

        mov     lr, r3, lsr #8

        ldmdb   r1!, {r3-r5, r12}

        orr     lr, lr, r12, lsl #24

        mov     r12, r12, lsr #8

        orr     r12, r12, r5, lsl #24

        mov     r5, r5, lsr #8

        orr     r5, r5, r4, lsl #24

        mov     r4, r4, lsr #8

        orr     r4, r4, r3, lsl #24

#else

        mov     lr, r3, lsl #8

        ldmdb   r1!, {r3-r5, r12}

        orr     lr, lr, r12, lsr #24

        mov     r12, r12, lsl #8

        orr     r12, r12, r5, lsr #24

        mov     r5, r5, lsl #8

        orr     r5, r5, r4, lsr #24

        mov     r4, r4, lsl #8

        orr     r4, r4, r3, lsr #24

#endif

        stmdb   r0!, {r4, r5, r12, lr}

        subs    r2, r2, #0x10

        bge     .Lmemcpy_bsrcul3loop16

        ldmia   sp!, {r4, r5, lr}

        adds    r2, r2, #0x0c

        blt     .Lmemcpy_bsrcul3l4

.Lmemcpy_bsrcul3loop4:

#if __BYTE_ORDER == __BIG_ENDIAN

        mov     r12, r3, lsr #8

        ldr     r3, [r1, #-4]!

        orr     r12, r12, r3, lsl #24

#else

        mov     r12, r3, lsl #8

        ldr     r3, [r1, #-4]!

        orr     r12, r12, r3, lsr #24

#endif

        str     r12, [r0, #-4]!

        subs    r2, r2, #4

        bge     .Lmemcpy_bsrcul3loop4

.Lmemcpy_bsrcul3l4:

        add     r1, r1, #3

        b       .Lmemcpy_bl4

.Lmemcpy_bsrcul2:

        cmp     r2, #0x0c

        blt     .Lmemcpy_bsrcul2loop4

        sub     r2, r2, #0x0c

        stmdb   sp!, {r4, r5, lr}

.Lmemcpy_bsrcul2loop16:

#if __BYTE_ORDER == __BIG_ENDIAN

        mov     lr, r3, lsr #16

        ldmdb   r1!, {r3-r5, r12}

        orr     lr, lr, r12, lsl #16

        mov     r12, r12, lsr #16

        orr     r12, r12, r5, lsl #16

        mov     r5, r5, lsr #16

        orr     r5, r5, r4, lsl #16

        mov     r4, r4, lsr #16

        orr     r4, r4, r3, lsl #16

#else

        mov     lr, r3, lsl #16

        ldmdb   r1!, {r3-r5, r12}

        orr     lr, lr, r12, lsr #16

        mov     r12, r12, lsl #16

        orr     r12, r12, r5, lsr #16

        mov     r5, r5, lsl #16

        orr     r5, r5, r4, lsr #16

        mov     r4, r4, lsl #16

        orr     r4, r4, r3, lsr #16

#endif

        stmdb   r0!, {r4, r5, r12, lr}

        subs    r2, r2, #0x10

        bge     .Lmemcpy_bsrcul2loop16

        ldmia   sp!, {r4, r5, lr}

        adds    r2, r2, #0x0c

        blt     .Lmemcpy_bsrcul2l4

.Lmemcpy_bsrcul2loop4:

#if __BYTE_ORDER == __BIG_ENDIAN

        mov     r12, r3, lsr #16

        ldr     r3, [r1, #-4]!

        orr     r12, r12, r3, lsl #16

#else

        mov     r12, r3, lsl #16

        ldr     r3, [r1, #-4]!

        orr     r12, r12, r3, lsr #16

#endif

        str     r12, [r0, #-4]!

        subs    r2, r2, #4

        bge     .Lmemcpy_bsrcul2loop4

.Lmemcpy_bsrcul2l4:

        add     r1, r1, #2

        b       .Lmemcpy_bl4

.Lmemcpy_bsrcul1:

        cmp     r2, #0x0c

        blt     .Lmemcpy_bsrcul1loop4

        sub     r2, r2, #0x0c

        stmdb   sp!, {r4, r5, lr}

.Lmemcpy_bsrcul1loop32:

#if __BYTE_ORDER == __BIG_ENDIAN

        mov     lr, r3, lsr #24

        ldmdb   r1!, {r3-r5, r12}

        orr     lr, lr, r12, lsl #8

        mov     r12, r12, lsr #24

        orr     r12, r12, r5, lsl #8

        mov     r5, r5, lsr #24

        orr     r5, r5, r4, lsl #8

        mov     r4, r4, lsr #24

        orr     r4, r4, r3, lsl #8

#else

        mov     lr, r3, lsl #24

        ldmdb   r1!, {r3-r5, r12}

        orr     lr, lr, r12, lsr #8

        mov     r12, r12, lsl #24

        orr     r12, r12, r5, lsr #8

        mov     r5, r5, lsl #24

        orr     r5, r5, r4, lsr #8

        mov     r4, r4, lsl #24

        orr     r4, r4, r3, lsr #8

#endif

        stmdb   r0!, {r4, r5, r12, lr}

        subs    r2, r2, #0x10

        bge     .Lmemcpy_bsrcul1loop32

        ldmia   sp!, {r4, r5, lr}

        adds    r2, r2, #0x0c

        blt     .Lmemcpy_bsrcul1l4

.Lmemcpy_bsrcul1loop4:

#if __BYTE_ORDER == __BIG_ENDIAN

        mov     r12, r3, lsr #24

        ldr     r3, [r1, #-4]!

        orr     r12, r12, r3, lsl #8

#else

        mov     r12, r3, lsl #24

        ldr     r3, [r1, #-4]!

        orr     r12, r12, r3, lsr #8

#endif

        str     r12, [r0, #-4]!

        subs    r2, r2, #4

        bge     .Lmemcpy_bsrcul1loop4

.Lmemcpy_bsrcul1l4:

        add     r1, r1, #1

        b       .Lmemcpy_bl4