Subversion Repositories or1k

/*
 * arch/alpha/lib/ev6-memset.S
 *
 * This is an efficient (and relatively small) implementation of the C library
 * "memset()" function for the 21264 implementation of Alpha.
 *
 * 21264 version  contributed by Rick Gorton <rick.gorton@alpha-processor.com>
 *
 * Much of the information about 21264 scheduling/coding comes from:
 *      Compiler Writer's Guide for the Alpha 21264
 *      abbreviated as 'CWG' in other comments here
 *      ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
 * Scheduling notation:
 *      E       - either cluster
 *      U       - upper subcluster; U0 - subcluster U0; U1 - subcluster U1
 *      L       - lower subcluster; L0 - subcluster L0; L1 - subcluster L1
 * The algorithm for the leading and trailing quadwords remains the same,
 * however the loop has been unrolled to enable better memory throughput,
 * and the code has been replicated for each of the entry points: __memset
 * and __memsetw to permit better scheduling to eliminate the stalling
 * encountered during the mask replication.
 * A future enhancement might be to put in a byte store loop for really
 * small (say < 32 bytes) memset()s.  Whether or not that change would be
 * a win in the kernel would depend upon the contextual usage.
 * WARNING: Maintaining this is going to be more work than the above version,
 * as fixes will need to be made in multiple places.  The performance gain
 * is worth it.
 */

        .set noat
        .set noreorder
.text
        .globl __memset
        .globl __memsetw
        .globl __constant_c_memset
        .globl memset

        .ent __memset
.align 5
__memset:
memset:
        .frame $30,0,$26,0
        .prologue 0

        /*
         * Serious stalling happens.  The only way to mitigate this is to
         * undertake a major re-write to interleave the constant materialization
         * with other parts of the fall-through code.  This is important, even
         * though it makes maintenance tougher.
         * Do this later.
         */
        and $17,255,$1          # E : 00000000000000ch
        insbl $17,1,$2          # U : 000000000000ch00
        bis $16,$16,$0          # E : return value
        ble $18,end_b           # U : zero length requested?

        addq $18,$16,$6         # E : max address to write to
        bis     $1,$2,$17       # E : 000000000000chch
        insbl   $1,2,$3         # U : 0000000000ch0000
        insbl   $1,3,$4         # U : 00000000ch000000

        or      $3,$4,$3        # E : 00000000chch0000
        inswl   $17,4,$5        # U : 0000chch00000000
        xor     $16,$6,$1       # E : will complete write be within one quadword?
        inswl   $17,6,$2        # U : chch000000000000

        or      $17,$3,$17      # E : 00000000chchchch
        or      $2,$5,$2        # E : chchchch00000000
        bic     $1,7,$1         # E : fit within a single quadword?
        and     $16,7,$3        # E : Target addr misalignment

        or      $17,$2,$17      # E : chchchchchchchch
        beq     $1,within_quad_b # U :
        nop                     # E :
        beq     $3,aligned_b    # U : target is 0mod8

        /*
         * Target address is misaligned, and won't fit within a quadword
         */
        ldq_u $4,0($16)         # L : Fetch first partial
        bis $16,$16,$5          # E : Save the address
        insql $17,$16,$2        # U : Insert new bytes
        subq $3,8,$3            # E : Invert (for addressing uses)

        addq $18,$3,$18         # E : $18 is new count ($3 is negative)
        mskql $4,$16,$4         # U : clear relevant parts of the quad
        subq $16,$3,$16         # E : $16 is new aligned destination
        bis $2,$4,$1            # E : Final bytes

        nop
        stq_u $1,0($5)          # L : Store result
        nop
        nop

.align 4
aligned_b:
        /*
         * We are now guaranteed to be quad aligned, with at least
         * one partial quad to write.
         */

        sra $18,3,$3            # U : Number of remaining quads to write
        and $18,7,$18           # E : Number of trailing bytes to write
        bis $16,$16,$5          # E : Save dest address
        beq $3,no_quad_b        # U : tail stuff only

        /*
         * it's worth the effort to unroll this and use wh64 if possible
         * Lifted a bunch of code from clear_user.S
         * At this point, entry values are:
         * $16  Current destination address
         * $5   A copy of $16
         * $6   The max quadword address to write to
         * $18  Number trailer bytes
         * $3   Number quads to write
         */

        and     $16, 0x3f, $2   # E : Forward work (only useful for unrolled loop)
        subq    $3, 16, $4      # E : Only try to unroll if > 128 bytes
        subq    $2, 0x40, $1    # E : bias counter (aligning stuff 0mod64)
        blt     $4, loop_b      # U :

        /*
         * We know we've got at least 16 quads, minimum of one trip
         * through unrolled loop.  Do a quad at a time to get us 0mod64
         * aligned.
         */

        nop                     # E :
        nop                     # E :
        nop                     # E :
        beq     $1, $bigalign_b # U :

$alignmod64_b:
        stq     $17, 0($5)      # L :
        subq    $3, 1, $3       # E : For consistency later
        addq    $1, 8, $1       # E : Increment towards zero for alignment
        addq    $5, 8, $4       # E : Initial wh64 address (filler instruction)

        nop
        nop
        addq    $5, 8, $5       # E : Inc address
        blt     $1, $alignmod64_b # U :

$bigalign_b:
        /*
         * $3 - number quads left to go
         * $5 - target address (aligned 0mod64)
         * $17 - mask of stuff to store
         * Scratch registers available: $7, $2, $4, $1
         * we know that we'll be taking a minimum of one trip through
         * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle
         * Assumes the wh64 needs to be for 2 trips through the loop in the future
         * The wh64 is issued on for the starting destination address for trip +2
         * through the loop, and if there are less than two trips left, the target
         * address will be for the current trip.
         */

$do_wh64_b:
        wh64    ($4)            # L1 : memory subsystem write hint
        subq    $3, 24, $2      # E : For determining future wh64 addresses
        stq     $17, 0($5)      # L :
        nop                     # E :

        addq    $5, 128, $4     # E : speculative target of next wh64
        stq     $17, 8($5)      # L :
        stq     $17, 16($5)     # L :
        addq    $5, 64, $7      # E : Fallback address for wh64 (== next trip addr)

        stq     $17, 24($5)     # L :
        stq     $17, 32($5)     # L :
        cmovlt  $2, $7, $4      # E : Latency 2, extra mapping cycle
        nop

        stq     $17, 40($5)     # L :
        stq     $17, 48($5)     # L :
        subq    $3, 16, $2      # E : Repeat the loop at least once more?
        nop

        stq     $17, 56($5)     # L :
        addq    $5, 64, $5      # E :
        subq    $3, 8, $3       # E :
        bge     $2, $do_wh64_b  # U :

        nop
        nop
        nop
        beq     $3, no_quad_b   # U : Might have finished already

.align 4
        /*
         * Simple loop for trailing quadwords, or for small amounts
         * of data (where we can't use an unrolled loop and wh64)
         */
loop_b:
        stq $17,0($5)           # L :
        subq $3,1,$3            # E : Decrement number quads left
        addq $5,8,$5            # E : Inc address
        bne $3,loop_b           # U : more?

no_quad_b:
        /*
         * Write 0..7 trailing bytes.
         */
        nop                     # E :
        beq $18,end_b           # U : All done?
        ldq $7,0($5)            # L :
        mskqh $7,$6,$2          # U : Mask final quad

        insqh $17,$6,$4         # U : New bits
        bis $2,$4,$1            # E : Put it all together
        stq $1,0($5)            # L : And back to memory
        ret $31,($26),1         # L0 :

within_quad_b:
        ldq_u $1,0($16)         # L :
        insql $17,$16,$2        # U : New bits
        mskql $1,$16,$4         # U : Clear old
        bis $2,$4,$2            # E : New result

        mskql $2,$6,$4          # U :
        mskqh $1,$6,$2          # U :
        bis $2,$4,$1            # E :
        stq_u $1,0($16)         # L :

end_b:
        nop
        nop
        nop
        ret $31,($26),1         # L0 :
        .end __memset

        /*
         * This is the original body of code, prior to replication and
         * rescheduling.  Leave it here, as there may be calls to this
         * entry point.
         */
.align 4
        .ent __constant_c_memset
__constant_c_memset:
        .frame $30,0,$26,0
        .prologue 0

        addq $18,$16,$6         # E : max address to write to
        bis $16,$16,$0          # E : return value
        xor $16,$6,$1           # E : will complete write be within one quadword?
        ble $18,end             # U : zero length requested?

        bic $1,7,$1             # E : fit within a single quadword
        beq $1,within_one_quad  # U :
        and $16,7,$3            # E : Target addr misalignment
        beq $3,aligned          # U : target is 0mod8

        /*
         * Target address is misaligned, and won't fit within a quadword
         */
        ldq_u $4,0($16)         # L : Fetch first partial
        bis $16,$16,$5          # E : Save the address
        insql $17,$16,$2        # U : Insert new bytes
        subq $3,8,$3            # E : Invert (for addressing uses)

        addq $18,$3,$18         # E : $18 is new count ($3 is negative)
        mskql $4,$16,$4         # U : clear relevant parts of the quad
        subq $16,$3,$16         # E : $16 is new aligned destination
        bis $2,$4,$1            # E : Final bytes

        nop
        stq_u $1,0($5)          # L : Store result
        nop
        nop

.align 4
aligned:
        /*
         * We are now guaranteed to be quad aligned, with at least
         * one partial quad to write.
         */

        sra $18,3,$3            # U : Number of remaining quads to write
        and $18,7,$18           # E : Number of trailing bytes to write
        bis $16,$16,$5          # E : Save dest address
        beq $3,no_quad          # U : tail stuff only

        /*
         * it's worth the effort to unroll this and use wh64 if possible
         * Lifted a bunch of code from clear_user.S
         * At this point, entry values are:
         * $16  Current destination address
         * $5   A copy of $16
         * $6   The max quadword address to write to
         * $18  Number trailer bytes
         * $3   Number quads to write
         */

        and     $16, 0x3f, $2   # E : Forward work (only useful for unrolled loop)
        subq    $3, 16, $4      # E : Only try to unroll if > 128 bytes
        subq    $2, 0x40, $1    # E : bias counter (aligning stuff 0mod64)
        blt     $4, loop        # U :

        /*
         * We know we've got at least 16 quads, minimum of one trip
         * through unrolled loop.  Do a quad at a time to get us 0mod64
         * aligned.
         */

        nop                     # E :
        nop                     # E :
        nop                     # E :
        beq     $1, $bigalign   # U :

$alignmod64:
        stq     $17, 0($5)      # L :
        subq    $3, 1, $3       # E : For consistency later
        addq    $1, 8, $1       # E : Increment towards zero for alignment
        addq    $5, 8, $4       # E : Initial wh64 address (filler instruction)

        nop
        nop
        addq    $5, 8, $5       # E : Inc address
        blt     $1, $alignmod64 # U :

$bigalign:
        /*
         * $3 - number quads left to go
         * $5 - target address (aligned 0mod64)
         * $17 - mask of stuff to store
         * Scratch registers available: $7, $2, $4, $1
         * we know that we'll be taking a minimum of one trip through
         * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle
         * Assumes the wh64 needs to be for 2 trips through the loop in the future
         * The wh64 is issued on for the starting destination address for trip +2
         * through the loop, and if there are less than two trips left, the target
         * address will be for the current trip.
         */

$do_wh64:
        wh64    ($4)            # L1 : memory subsystem write hint
        subq    $3, 24, $2      # E : For determining future wh64 addresses
        stq     $17, 0($5)      # L :
        nop                     # E :

        addq    $5, 128, $4     # E : speculative target of next wh64
        stq     $17, 8($5)      # L :
        stq     $17, 16($5)     # L :
        addq    $5, 64, $7      # E : Fallback address for wh64 (== next trip addr)

        stq     $17, 24($5)     # L :
        stq     $17, 32($5)     # L :
        cmovlt  $2, $7, $4      # E : Latency 2, extra mapping cycle
        nop

        stq     $17, 40($5)     # L :
        stq     $17, 48($5)     # L :
        subq    $3, 16, $2      # E : Repeat the loop at least once more?
        nop

        stq     $17, 56($5)     # L :
        addq    $5, 64, $5      # E :
        subq    $3, 8, $3       # E :
        bge     $2, $do_wh64    # U :

        nop
        nop
        nop
        beq     $3, no_quad     # U : Might have finished already

.align 4
        /*
         * Simple loop for trailing quadwords, or for small amounts
         * of data (where we can't use an unrolled loop and wh64)
         */
loop:
        stq $17,0($5)           # L :
        subq $3,1,$3            # E : Decrement number quads left
        addq $5,8,$5            # E : Inc address
        bne $3,loop             # U : more?

no_quad:
        /*
         * Write 0..7 trailing bytes.
         */
        nop                     # E :
        beq $18,end             # U : All done?
        ldq $7,0($5)            # L :
        mskqh $7,$6,$2          # U : Mask final quad

        insqh $17,$6,$4         # U : New bits
        bis $2,$4,$1            # E : Put it all together
        stq $1,0($5)            # L : And back to memory
        ret $31,($26),1         # L0 :

within_one_quad:
        ldq_u $1,0($16)         # L :
        insql $17,$16,$2        # U : New bits
        mskql $1,$16,$4         # U : Clear old
        bis $2,$4,$2            # E : New result

        mskql $2,$6,$4          # U :
        mskqh $1,$6,$2          # U :
        bis $2,$4,$1            # E :
        stq_u $1,0($16)         # L :

end:
        nop
        nop
        nop
        ret $31,($26),1         # L0 :
        .end __constant_c_memset

        /*
         * This is a replicant of the __constant_c_memset code, rescheduled
         * to mask stalls.  Note that entry point names also had to change
         */
        .align 5
        .ent __memsetw

__memsetw:
        .frame $30,0,$26,0
        .prologue 0

        inswl $17,0,$5          # U : 000000000000c1c2
        inswl $17,2,$2          # U : 00000000c1c20000
        bis $16,$16,$0          # E : return value
        addq    $18,$16,$6      # E : max address to write to

        ble $18, end_w          # U : zero length requested?
        inswl   $17,4,$3        # U : 0000c1c200000000
        inswl   $17,6,$4        # U : c1c2000000000000
        xor     $16,$6,$1       # E : will complete write be within one quadword?

        or      $2,$5,$2        # E : 00000000c1c2c1c2
        or      $3,$4,$17       # E : c1c2c1c200000000
        bic     $1,7,$1         # E : fit within a single quadword
        and     $16,7,$3        # E : Target addr misalignment

        or      $17,$2,$17      # E : c1c2c1c2c1c2c1c2
        beq $1,within_quad_w    # U :
        nop
        beq $3,aligned_w        # U : target is 0mod8

        /*
         * Target address is misaligned, and won't fit within a quadword
         */
        ldq_u $4,0($16)         # L : Fetch first partial
        bis $16,$16,$5          # E : Save the address
        insql $17,$16,$2        # U : Insert new bytes
        subq $3,8,$3            # E : Invert (for addressing uses)

        addq $18,$3,$18         # E : $18 is new count ($3 is negative)
        mskql $4,$16,$4         # U : clear relevant parts of the quad
        subq $16,$3,$16         # E : $16 is new aligned destination
        bis $2,$4,$1            # E : Final bytes

        nop
        stq_u $1,0($5)          # L : Store result
        nop
        nop

.align 4
aligned_w:
        /*
         * We are now guaranteed to be quad aligned, with at least
         * one partial quad to write.
         */

        sra $18,3,$3            # U : Number of remaining quads to write
        and $18,7,$18           # E : Number of trailing bytes to write
        bis $16,$16,$5          # E : Save dest address
        beq $3,no_quad_w        # U : tail stuff only

        /*
         * it's worth the effort to unroll this and use wh64 if possible
         * Lifted a bunch of code from clear_user.S
         * At this point, entry values are:
         * $16  Current destination address
         * $5   A copy of $16
         * $6   The max quadword address to write to
         * $18  Number trailer bytes
         * $3   Number quads to write
         */

        and     $16, 0x3f, $2   # E : Forward work (only useful for unrolled loop)
        subq    $3, 16, $4      # E : Only try to unroll if > 128 bytes
        subq    $2, 0x40, $1    # E : bias counter (aligning stuff 0mod64)
        blt     $4, loop_w      # U :

        /*
         * We know we've got at least 16 quads, minimum of one trip
         * through unrolled loop.  Do a quad at a time to get us 0mod64
         * aligned.
         */

        nop                     # E :
        nop                     # E :
        nop                     # E :
        beq     $1, $bigalign_w # U :

$alignmod64_w:
        stq     $17, 0($5)      # L :
        subq    $3, 1, $3       # E : For consistency later
        addq    $1, 8, $1       # E : Increment towards zero for alignment
        addq    $5, 8, $4       # E : Initial wh64 address (filler instruction)

        nop
        nop
        addq    $5, 8, $5       # E : Inc address
        blt     $1, $alignmod64_w       # U :

$bigalign_w:
        /*
         * $3 - number quads left to go
         * $5 - target address (aligned 0mod64)
         * $17 - mask of stuff to store
         * Scratch registers available: $7, $2, $4, $1
         * we know that we'll be taking a minimum of one trip through
         * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle
         * Assumes the wh64 needs to be for 2 trips through the loop in the future
         * The wh64 is issued on for the starting destination address for trip +2
         * through the loop, and if there are less than two trips left, the target
         * address will be for the current trip.
         */

$do_wh64_w:
        wh64    ($4)            # L1 : memory subsystem write hint
        subq    $3, 24, $2      # E : For determining future wh64 addresses
        stq     $17, 0($5)      # L :
        nop                     # E :

        addq    $5, 128, $4     # E : speculative target of next wh64
        stq     $17, 8($5)      # L :
        stq     $17, 16($5)     # L :
        addq    $5, 64, $7      # E : Fallback address for wh64 (== next trip addr)

        stq     $17, 24($5)     # L :
        stq     $17, 32($5)     # L :
        cmovlt  $2, $7, $4      # E : Latency 2, extra mapping cycle
        nop

        stq     $17, 40($5)     # L :
        stq     $17, 48($5)     # L :
        subq    $3, 16, $2      # E : Repeat the loop at least once more?
        nop

        stq     $17, 56($5)     # L :
        addq    $5, 64, $5      # E :
        subq    $3, 8, $3       # E :
        bge     $2, $do_wh64_w  # U :

        nop
        nop
        nop
        beq     $3, no_quad_w   # U : Might have finished already

.align 4
        /*
         * Simple loop for trailing quadwords, or for small amounts
         * of data (where we can't use an unrolled loop and wh64)
         */
loop_w:
        stq $17,0($5)           # L :
        subq $3,1,$3            # E : Decrement number quads left
        addq $5,8,$5            # E : Inc address
        bne $3,loop_w           # U : more?

no_quad_w:
        /*
         * Write 0..7 trailing bytes.
         */
        nop                     # E :
        beq $18,end_w           # U : All done?
        ldq $7,0($5)            # L :
        mskqh $7,$6,$2          # U : Mask final quad

        insqh $17,$6,$4         # U : New bits
        bis $2,$4,$1            # E : Put it all together
        stq $1,0($5)            # L : And back to memory
        ret $31,($26),1         # L0 :

within_quad_w:
        ldq_u $1,0($16)         # L :
        insql $17,$16,$2        # U : New bits
        mskql $1,$16,$4         # U : Clear old
        bis $2,$4,$2            # E : New result

        mskql $2,$6,$4          # U :
        mskqh $1,$6,$2          # U :
        bis $2,$4,$1            # E :
        stq_u $1,0($16)         # L :

end_w:
        nop
        nop
        nop
        ret $31,($26),1         # L0 :

        .end __memsetw