Subversion Repositories or1k

/* $Id: blockops.S,v 1.1.1.1 2004-04-15 01:33:46 phoenix Exp $
 * blockops.S: UltraSparc block zero optimized routines.
 *
 * Copyright (C) 1996, 1998, 1999, 2000 David S. Miller (davem@redhat.com)
 * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com)
 */

#include "VIS.h"
#include <asm/visasm.h>
#include <asm/page.h>
#include <asm/dcu.h>
#include <asm/spitfire.h>
#include <asm/pgtable.h>
#include <asm/asm_offsets.h>

#define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7)   \
        fmovd   %reg0, %f48;    fmovd   %reg1, %f50;            \
        fmovd   %reg2, %f52;    fmovd   %reg3, %f54;            \
        fmovd   %reg4, %f56;    fmovd   %reg5, %f58;            \
        fmovd   %reg6, %f60;    fmovd   %reg7, %f62;

#define DCACHE_SIZE     (PAGE_SIZE * 2)
#define TLBTEMP_ENT1    (60 << 3)
#define TLBTEMP_ENT2    (61 << 3)
#define TLBTEMP_ENTSZ   (1 << 3)

#if (PAGE_SHIFT == 13) || (PAGE_SHIFT == 19)
#define PAGE_SIZE_REM   0x80
#elif (PAGE_SHIFT == 16) || (PAGE_SHIFT == 22)
#define PAGE_SIZE_REM   0x100
#else
#error Wrong PAGE_SHIFT specified
#endif

        .text

        .align          32
        .globl          copy_user_page
        .type           copy_user_page,@function
copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
        VISEntry
        sethi           %hi(PAGE_SIZE), %g3
        sub             %o0, %g4, %g1
        and             %o2, %g3, %o0
        sethi           %hi(TLBTEMP_BASE), %o3
        sethi           %uhi(_PAGE_VALID | _PAGE_SZBITS), %g3
        sub             %o1, %g4, %g2
        sllx            %g3, 32, %g3
        mov             TLB_TAG_ACCESS, %o2
        or              %g3, (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W), %g3
        sethi           %hi(DCACHE_SIZE), %o1
        or              %g1, %g3, %g1
        or              %g2, %g3, %g2
        add             %o0, %o3, %o0
        add             %o0, %o1, %o1
#define FIX_INSN_1      0x96102060 /* mov (12 << 3), %o3 */
cheetah_patch_1:
        mov             TLBTEMP_ENT1, %o3
        rdpr            %pstate, %g3
        wrpr            %g3, PSTATE_IE, %pstate

        /* Do this now, before loading the fixed TLB entries for copying,
         * so we do not risk a multiple TLB match condition later when
         * restoring those entries.
         */
        ldub            [%g6 + AOFF_task_thread + AOFF_thread_use_blkcommit], %g3

        /* Spitfire Errata #32 workaround */
        mov             PRIMARY_CONTEXT, %o4
        stxa            %g0, [%o4] ASI_DMMU
        membar          #Sync

        ldxa            [%o3] ASI_DTLB_TAG_READ, %o4

        /* Spitfire Errata #32 workaround */
        mov             PRIMARY_CONTEXT, %o5
        stxa            %g0, [%o5] ASI_DMMU
        membar          #Sync

        ldxa            [%o3] ASI_DTLB_DATA_ACCESS, %g0
        ldxa            [%o3] ASI_DTLB_DATA_ACCESS, %o5
        stxa            %o0, [%o2] ASI_DMMU
        stxa            %g1, [%o3] ASI_DTLB_DATA_ACCESS
        membar          #Sync
        add             %o3, (TLBTEMP_ENTSZ), %o3

        /* Spitfire Errata #32 workaround */
        mov             PRIMARY_CONTEXT, %g5
        stxa            %g0, [%g5] ASI_DMMU
        membar          #Sync

        ldxa            [%o3] ASI_DTLB_TAG_READ, %g5

        /* Spitfire Errata #32 workaround */
        mov             PRIMARY_CONTEXT, %g7
        stxa            %g0, [%g7] ASI_DMMU
        membar          #Sync

        ldxa            [%o3] ASI_DTLB_DATA_ACCESS, %g0
        ldxa            [%o3] ASI_DTLB_DATA_ACCESS, %g7
        stxa            %o1, [%o2] ASI_DMMU
        stxa            %g2, [%o3] ASI_DTLB_DATA_ACCESS
        membar          #Sync

        cmp             %g3, 0
        bne,pn          %xcc, copy_page_using_blkcommit
         nop

        BRANCH_IF_ANY_CHEETAH(g3,o2,cheetah_copy_user_page)
        ba,pt           %xcc, spitfire_copy_user_page
         nop

cheetah_copy_user_page:
        .globl          cheetah_copy_user_page_nop_1_6
cheetah_copy_user_page_nop_1_6:
        ldxa            [%g0] ASI_DCU_CONTROL_REG, %g3
        sethi           %uhi(DCU_PE), %o2
        sllx            %o2, 32, %o2
        or              %g3, %o2, %o2
        stxa            %o2, [%g0] ASI_DCU_CONTROL_REG  ! Enable P-cache
        membar          #Sync

        sethi           %hi((PAGE_SIZE/64)-7), %o2      ! A0 Group
        prefetch        [%o1 + 0x000], #one_read        ! MS
        or              %o2, %lo((PAGE_SIZE/64)-7), %o2 ! A1 Group
        prefetch        [%o1 + 0x040], #one_read        ! MS
        prefetch        [%o1 + 0x080], #one_read        ! MS Group
        prefetch        [%o1 + 0x0c0], #one_read        ! MS Group
        ldd             [%o1 + 0x000], %f0              ! MS Group
        prefetch        [%o1 + 0x100], #one_read        ! MS Group
        ldd             [%o1 + 0x008], %f2              ! AX
        prefetch        [%o1 + 0x140], #one_read        ! MS Group
        ldd             [%o1 + 0x010], %f4              ! AX
        prefetch        [%o1 + 0x180], #one_read        ! MS Group
        fmovd           %f0, %f32                       ! FGA Group
        ldd             [%o1 + 0x018], %f6              ! AX
        fmovd           %f2, %f34                       ! FGA Group
        ldd             [%o1 + 0x020], %f8              ! MS
        fmovd           %f4, %f36                       ! FGA Group
        ldd             [%o1 + 0x028], %f10             ! AX
        membar          #StoreStore                     ! MS
        fmovd           %f6, %f38                       ! FGA Group
        ldd             [%o1 + 0x030], %f12             ! MS
        fmovd           %f8, %f40                       ! FGA Group
        ldd             [%o1 + 0x038], %f14             ! AX
        fmovd           %f10, %f42                      ! FGA Group
        ldd             [%o1 + 0x040], %f16             ! MS
1:      ldd             [%o1 + 0x048], %f2              ! AX (Group)
        fmovd           %f12, %f44                      ! FGA
        ldd             [%o1 + 0x050], %f4              ! MS
        fmovd           %f14, %f46                      ! FGA Group
        stda            %f32, [%o0] ASI_BLK_P           ! MS
        ldd             [%o1 + 0x058], %f6              ! AX
        fmovd           %f16, %f32                      ! FGA Group (8-cycle stall)
        ldd             [%o1 + 0x060], %f8              ! MS
        fmovd           %f2, %f34                       ! FGA Group
        ldd             [%o1 + 0x068], %f10             ! AX
        fmovd           %f4, %f36                       ! FGA Group
        ldd             [%o1 + 0x070], %f12             ! MS
        fmovd           %f6, %f38                       ! FGA Group
        ldd             [%o1 + 0x078], %f14             ! AX
        fmovd           %f8, %f40                       ! FGA Group
        ldd             [%o1 + 0x080], %f16             ! AX
        prefetch        [%o1 + 0x180], #one_read        ! MS
        fmovd           %f10, %f42                      ! FGA Group
        subcc           %o2, 1, %o2                     ! A0
        add             %o0, 0x40, %o0                  ! A1
        bne,pt          %xcc, 1b                        ! BR
         add            %o1, 0x40, %o1                  ! A0 Group

        mov             5, %o2                          ! A0 Group
1:      ldd             [%o1 + 0x048], %f2              ! AX
        fmovd           %f12, %f44                      ! FGA
        ldd             [%o1 + 0x050], %f4              ! MS
        fmovd           %f14, %f46                      ! FGA Group
        stda            %f32, [%o0] ASI_BLK_P           ! MS
        ldd             [%o1 + 0x058], %f6              ! AX
        fmovd           %f16, %f32                      ! FGA Group (8-cycle stall)
        ldd             [%o1 + 0x060], %f8              ! MS
        fmovd           %f2, %f34                       ! FGA Group
        ldd             [%o1 + 0x068], %f10             ! AX
        fmovd           %f4, %f36                       ! FGA Group
        ldd             [%o1 + 0x070], %f12             ! MS
        fmovd           %f6, %f38                       ! FGA Group
        ldd             [%o1 + 0x078], %f14             ! AX
        fmovd           %f8, %f40                       ! FGA Group
        ldd             [%o1 + 0x080], %f16             ! MS
        fmovd           %f10, %f42                      ! FGA Group
        subcc           %o2, 1, %o2                     ! A0
        add             %o0, 0x40, %o0                  ! A1
        bne,pt          %xcc, 1b                        ! BR
         add            %o1, 0x40, %o1                  ! A0 Group

        ldd             [%o1 + 0x048], %f2              ! AX
        fmovd           %f12, %f44                      ! FGA
        ldd             [%o1 + 0x050], %f4              ! MS
        fmovd           %f14, %f46                      ! FGA Group
        stda            %f32, [%o0] ASI_BLK_P           ! MS
        ldd             [%o1 + 0x058], %f6              ! AX
        fmovd           %f16, %f32                      ! FGA Group (8-cycle stall)
        ldd             [%o1 + 0x060], %f8              ! MS
        fmovd           %f2, %f34                       ! FGA Group
        ldd             [%o1 + 0x068], %f10             ! AX
        fmovd           %f4, %f36                       ! FGA Group
        ldd             [%o1 + 0x070], %f12             ! MS
        fmovd           %f6, %f38                       ! FGA Group
        add             %o0, 0x40, %o0                  ! A0
        ldd             [%o1 + 0x078], %f14             ! AX
        fmovd           %f8, %f40                       ! FGA Group
        fmovd           %f10, %f42                      ! FGA Group
        fmovd           %f12, %f44                      ! FGA Group
        fmovd           %f14, %f46                      ! FGA Group
        stda            %f32, [%o0] ASI_BLK_P           ! MS
        .globl          cheetah_copy_user_page_nop_2_3
cheetah_copy_user_page_nop_2_3:
        mov             PRIMARY_CONTEXT, %o2
        stxa            %g0, [%o2] ASI_DMMU             ! Flush P-cache
        stxa            %g3, [%g0] ASI_DCU_CONTROL_REG  ! Disable P-cache
        ba,a,pt         %xcc, copy_user_page_continue

spitfire_copy_user_page:
        ldda            [%o1] ASI_BLK_P, %f0
        add             %o1, 0x40, %o1
        ldda            [%o1] ASI_BLK_P, %f16
        add             %o1, 0x40, %o1
        sethi           %hi(PAGE_SIZE), %o2
1:      TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
        ldda            [%o1] ASI_BLK_P, %f32
        stda            %f48, [%o0] ASI_BLK_P
        add             %o1, 0x40, %o1
        sub             %o2, 0x40, %o2
        add             %o0, 0x40, %o0
        TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
        ldda            [%o1] ASI_BLK_P, %f0
        stda            %f48, [%o0] ASI_BLK_P
        add             %o1, 0x40, %o1
        sub             %o2, 0x40, %o2
        add             %o0, 0x40, %o0
        TOUCH(f32, f34, f36, f38, f40, f42, f44, f46)
        ldda            [%o1] ASI_BLK_P, %f16
        stda            %f48, [%o0] ASI_BLK_P
        sub             %o2, 0x40, %o2
        add             %o1, 0x40, %o1
        cmp             %o2, PAGE_SIZE_REM
        bne,pt          %xcc, 1b
         add            %o0, 0x40, %o0
#if (PAGE_SHIFT == 16) || (PAGE_SHIFT == 22)
        TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
        ldda            [%o1] ASI_BLK_P, %f32
        stda            %f48, [%o0] ASI_BLK_P
        add             %o1, 0x40, %o1
        sub             %o2, 0x40, %o2
        add             %o0, 0x40, %o0
        TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
        ldda            [%o1] ASI_BLK_P, %f0
        stda            %f48, [%o0] ASI_BLK_P
        add             %o1, 0x40, %o1
        sub             %o2, 0x40, %o2
        add             %o0, 0x40, %o0
        membar          #Sync
        stda            %f32, [%o0] ASI_BLK_P
        add             %o0, 0x40, %o0
        stda            %f0, [%o0] ASI_BLK_P
#else
        membar          #Sync
        stda            %f0, [%o0] ASI_BLK_P
        add             %o0, 0x40, %o0
        stda            %f16, [%o0] ASI_BLK_P
#endif
copy_user_page_continue:
        membar          #Sync
        VISExit

        mov             TLB_TAG_ACCESS, %o2
        stxa            %g5, [%o2] ASI_DMMU
        stxa            %g7, [%o3] ASI_DTLB_DATA_ACCESS
        membar          #Sync
        sub             %o3, (TLBTEMP_ENTSZ), %o3
        stxa            %o4, [%o2] ASI_DMMU
        stxa            %o5, [%o3] ASI_DTLB_DATA_ACCESS
        membar          #Sync
        rdpr            %pstate, %g3
        jmpl            %o7 + 0x8, %g0
         wrpr           %g3, PSTATE_IE, %pstate

copy_page_using_blkcommit:
        membar          #LoadStore | #StoreStore | #StoreLoad
        ldda            [%o1] ASI_BLK_P, %f0
        add             %o1, 0x40, %o1
        ldda            [%o1] ASI_BLK_P, %f16
        add             %o1, 0x40, %o1
        sethi           %hi(PAGE_SIZE), %o2
1:      TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
        ldda            [%o1] ASI_BLK_P, %f32
        stda            %f48, [%o0] ASI_BLK_COMMIT_P
        add             %o1, 0x40, %o1
        sub             %o2, 0x40, %o2
        add             %o0, 0x40, %o0
        TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
        ldda            [%o1] ASI_BLK_P, %f0
        stda            %f48, [%o0] ASI_BLK_COMMIT_P
        add             %o1, 0x40, %o1
        sub             %o2, 0x40, %o2
        add             %o0, 0x40, %o0
        TOUCH(f32, f34, f36, f38, f40, f42, f44, f46)
        ldda            [%o1] ASI_BLK_P, %f16
        stda            %f48, [%o0] ASI_BLK_COMMIT_P
        sub             %o2, 0x40, %o2
        add             %o1, 0x40, %o1
        cmp             %o2, PAGE_SIZE_REM
        bne,pt          %xcc, 1b
         add            %o0, 0x40, %o0
#if (PAGE_SHIFT == 16) || (PAGE_SHIFT == 22)
        TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
        ldda            [%o1] ASI_BLK_P, %f32
        stda            %f48, [%o0] ASI_BLK_COMMIT_P
        add             %o1, 0x40, %o1
        sub             %o2, 0x40, %o2
        add             %o0, 0x40, %o0
        TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
        ldda            [%o1] ASI_BLK_P, %f0
        stda            %f48, [%o0] ASI_BLK_COMMIT_P
        add             %o1, 0x40, %o1
        sub             %o2, 0x40, %o2
        add             %o0, 0x40, %o0
        membar          #Sync
        stda            %f32, [%o0] ASI_BLK_COMMIT_P
        add             %o0, 0x40, %o0
        ba,pt           %xcc, copy_user_page_continue
         stda           %f0, [%o0] ASI_BLK_COMMIT_P
#else
        membar          #Sync
        stda            %f0, [%o0] ASI_BLK_COMMIT_P
        add             %o0, 0x40, %o0
        ba,pt           %xcc, copy_user_page_continue
         stda           %f16, [%o0] ASI_BLK_COMMIT_P
#endif

        .align          32
        .globl          _clear_page
        .type           _clear_page,@function
_clear_page:    /* %o0=dest */
        VISEntryHalf
        ba,pt           %xcc, clear_page_common
         clr            %o4

        .align          32
        .globl          clear_user_page
        .type           clear_user_page,@function
clear_user_page:        /* %o0=dest, %o1=vaddr */
        VISEntryHalf
        sethi           %hi(PAGE_SIZE), %g3
        sub             %o0, %g4, %g1
        and             %o1, %g3, %o0
        mov             TLB_TAG_ACCESS, %o2
        sethi           %uhi(_PAGE_VALID | _PAGE_SZBITS), %g3
        sethi           %hi(TLBTEMP_BASE), %o3
        sllx            %g3, 32, %g3
        or              %g3, (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W), %g3
        or              %g1, %g3, %g1
        add             %o0, %o3, %o0
#define FIX_INSN_2      0x96102068 /* mov (13 << 3), %o3 */
cheetah_patch_2:
        mov             TLBTEMP_ENT2, %o3
        rdpr            %pstate, %g3
        wrpr            %g3, PSTATE_IE, %pstate

        /* Spitfire Errata #32 workaround */
        mov             PRIMARY_CONTEXT, %g5
        stxa            %g0, [%g5] ASI_DMMU
        membar          #Sync

        ldxa            [%o3] ASI_DTLB_TAG_READ, %g5

        /* Spitfire Errata #32 workaround */
        mov             PRIMARY_CONTEXT, %g7
        stxa            %g0, [%g7] ASI_DMMU
        membar          #Sync

        ldxa            [%o3] ASI_DTLB_DATA_ACCESS, %g0
        ldxa            [%o3] ASI_DTLB_DATA_ACCESS, %g7
        stxa            %o0, [%o2] ASI_DMMU
        stxa            %g1, [%o3] ASI_DTLB_DATA_ACCESS
        membar          #Sync

        mov             1, %o4

clear_page_common:
        membar          #StoreLoad | #StoreStore | #LoadStore   ! LSU   Group
        fzero           %f0                             ! FPA   Group
        sethi           %hi(PAGE_SIZE/256), %o1         ! IEU0
        fzero           %f2                             ! FPA   Group
        or              %o1, %lo(PAGE_SIZE/256), %o1    ! IEU0
        faddd           %f0, %f2, %f4                   ! FPA   Group
        fmuld           %f0, %f2, %f6                   ! FPM
        faddd           %f0, %f2, %f8                   ! FPA   Group
        fmuld           %f0, %f2, %f10                  ! FPM

        faddd           %f0, %f2, %f12                  ! FPA   Group
        fmuld           %f0, %f2, %f14                  ! FPM
1:      stda            %f0, [%o0 + %g0] ASI_BLK_P      ! Store Group
        add             %o0, 0x40, %o0                  ! IEU0
        stda            %f0, [%o0 + %g0] ASI_BLK_P      ! Store Group
        add             %o0, 0x40, %o0                  ! IEU0
        stda            %f0, [%o0 + %g0] ASI_BLK_P      ! Store Group

        add             %o0, 0x40, %o0                  ! IEU0  Group
        stda            %f0, [%o0 + %g0] ASI_BLK_P      ! Store Group
        subcc           %o1, 1, %o1                     ! IEU1
        bne,pt          %icc, 1b                        ! CTI
         add            %o0, 0x40, %o0                  ! IEU0  Group
        membar          #Sync                           ! LSU   Group
        VISExitHalf

        brnz,pt         %o4, 1f
         nop

        retl
         nop

1:
        stxa            %g5, [%o2] ASI_DMMU
        stxa            %g7, [%o3] ASI_DTLB_DATA_ACCESS
        membar          #Sync
        jmpl            %o7 + 0x8, %g0
         wrpr           %g3, 0x0, %pstate

        .globl          cheetah_patch_pgcopyops
cheetah_patch_pgcopyops:
        sethi           %hi(FIX_INSN_1), %g1
        or              %g1, %lo(FIX_INSN_1), %g1
        sethi           %hi(cheetah_patch_1), %g2
        or              %g2, %lo(cheetah_patch_1), %g2
        stw             %g1, [%g2]
        flush           %g2
        sethi           %hi(FIX_INSN_2), %g1
        or              %g1, %lo(FIX_INSN_2), %g1
        sethi           %hi(cheetah_patch_2), %g2
        or              %g2, %lo(cheetah_patch_2), %g2
        stw             %g1, [%g2]
        flush           %g2
        retl
         nop

#undef FIX_INSN1
#undef FIX_INSN2
#undef PAGE_SIZE_REM