Subversion Repositories or1k

/* Copyright 2002 Andi Kleen, SuSE Labs.

 * Subject to the GNU Public License v2.

 *

 * Functions to copy from and to user space.

 */

        #include 

/* #define FIX_ALIGNMENT 1 */

        #include 

        #include 

/* Standard copy_to_user with segment limit checking */

        .globl copy_to_user

        .p2align 4

copy_to_user:

        GET_CURRENT(%rax)

        movq %rdi,%rcx

        addq %rdx,%rcx

        jc  bad_to_user

        cmpq tsk_addr_limit(%rax),%rcx

        jae bad_to_user

        jmp copy_user_generic

/* Standard copy_from_user with segment limit checking */

        .globl copy_from_user

        .p2align 4

copy_from_user:

        GET_CURRENT(%rax)

        movq %rsi,%rcx

        addq %rdx,%rcx

        jc  bad_from_user

        cmpq tsk_addr_limit(%rax),%rcx

        jae  bad_from_user

        /* FALL THROUGH to copy_user_generic */

        .section .fixup,"ax"

        /* must zero dest */

bad_from_user:

        movl %edx,%ecx

        xorl %eax,%eax

        rep

        stosb

bad_to_user:

        movl    %edx,%eax

        ret

        .previous

/*

 * copy_user_generic - memory copy with exception handling.

 *

 * Input:

 * rdi destination

 * rsi source

 * rdx count

 *

 * Output:

 * eax uncopied bytes or 0 if successfull.

 */

        .globl copy_user_generic

        .p2align 4

copy_user_generic:

        prefetcht0 (%rsi)

#ifdef CONFIG_MK8

        prefetchw (%rdi)

#endif

        pushq %rbx

        xorl %eax,%eax          /*zero for the exception handler */

#ifdef FIX_ALIGNMENT

        /* check for bad alignment of destination */

        movl %edi,%ecx

        andl $7,%ecx

        jnz  .Lbad_alignment

.Lafter_bad_alignment:

#endif

        movq %rdx,%rcx

        movl $64,%ebx

        shrq $6,%rdx

        decq %rdx

        js   .Lhandle_tail

        .p2align 4

.Lloop:

.Ls1:   movq (%rsi),%r11

.Ls2:   movq 1*8(%rsi),%r8

.Ls3:   movq 2*8(%rsi),%r9

.Ls4:   movq 3*8(%rsi),%r10

.Ld1:   movq %r11,(%rdi)

.Ld2:   movq %r8,1*8(%rdi)

.Ld3:   movq %r9,2*8(%rdi)

.Ld4:   movq %r10,3*8(%rdi)

.Ls5:   movq 4*8(%rsi),%r11

.Ls6:   movq 5*8(%rsi),%r8

.Ls7:   movq 6*8(%rsi),%r9

.Ls8:   movq 7*8(%rsi),%r10

.Ld5:   movq %r11,4*8(%rdi)

.Ld6:   movq %r8,5*8(%rdi)

.Ld7:   movq %r9,6*8(%rdi)

.Ld8:   movq %r10,7*8(%rdi)

        decq %rdx

        leaq 64(%rsi),%rsi

        leaq 64(%rdi),%rdi

        jns  .Lloop

        .p2align 4

.Lhandle_tail:

        movl %ecx,%edx

        andl $63,%ecx

        shrl $3,%ecx

        jz   .Lhandle_7

        movl $8,%ebx

        .p2align 4

.Lloop_8:

.Ls9:   movq (%rsi),%r8

.Ld9:   movq %r8,(%rdi)

        decl %ecx

        leaq 8(%rdi),%rdi

        leaq 8(%rsi),%rsi

        jnz .Lloop_8

.Lhandle_7:

        movl %edx,%ecx

        andl $7,%ecx

        jz   .Lende

        .p2align 4

.Lloop_1:

.Ls10:  movb (%rsi),%bl

.Ld10:  movb %bl,(%rdi)

        incq %rdi

        incq %rsi

        decl %ecx

        jnz .Lloop_1

.Lende:

        popq %rbx

        ret

#ifdef FIX_ALIGNMENT

        /* align destination */

        .p2align 4

.Lbad_alignment:

        movl $8,%r9d

        subl %ecx,%r9d

        movl %r9d,%ecx

        subq %r9,%rdx

        jz   .Lsmall_align

        js   .Lsmall_align

.Lalign_1:

.Ls11:  movb (%rsi),%bl

.Ld11:  movb %bl,(%rdi)

        incq %rsi

        incq %rdi

        decl %ecx

        jnz .Lalign_1

        jmp .Lafter_bad_alignment

.Lsmall_align:

        addq %r9,%rdx

        jmp .Lhandle_7

#endif

        /* table sorted by exception address */

        .section __ex_table,"a"

        .align 8

        .quad .Ls1,.Ls1e

        .quad .Ls2,.Ls2e

        .quad .Ls3,.Ls3e

        .quad .Ls4,.Ls4e

        .quad .Ld1,.Ls1e

        .quad .Ld2,.Ls2e

        .quad .Ld3,.Ls3e

        .quad .Ld4,.Ls4e

        .quad .Ls5,.Ls5e

        .quad .Ls6,.Ls6e

        .quad .Ls7,.Ls7e

        .quad .Ls8,.Ls8e

        .quad .Ld5,.Ls5e

        .quad .Ld6,.Ls6e

        .quad .Ld7,.Ls7e

        .quad .Ld8,.Ls8e

        .quad .Ls9,.Le_quad

        .quad .Ld9,.Le_quad

        .quad .Ls10,.Le_byte

        .quad .Ld10,.Le_byte

#ifdef FIX_ALIGNMENT

        .quad .Ls11,.Le_byte

        .quad .Ld11,.Le_byte

#endif

        .quad .Le5,.Le_zero

        .previous

        /* compute 64-offset for main loop. 8 bytes accuracy with error on the

           pessimistic side. this is gross. it would be better to fix the

           interface. */

        /* eax: zero, ebx: 64 */

.Ls1e:  addl $8,%eax

.Ls2e:  addl $8,%eax

.Ls3e:  addl $8,%eax

.Ls4e:  addl $8,%eax

.Ls5e:  addl $8,%eax

.Ls6e:  addl $8,%eax

.Ls7e:  addl $8,%eax

.Ls8e:  addl $8,%eax

        addq %rbx,%rdi  /* +64 */

        subq %rax,%rdi  /* correct destination with computed offset */

        shlq $6,%rdx    /* loop counter * 64 (stride length) */

        addq %rax,%rdx  /* add offset to loopcnt */

        andl $63,%ecx   /* remaining bytes */

        addq %rcx,%rdx  /* add them */

        jmp .Lzero_rest

        /* exception on quad word loop in tail handling */

        /* ecx: loopcnt/8, %edx: length, rdi: correct */

.Le_quad:

        shll $3,%ecx

        andl $7,%edx

        addl %ecx,%edx

        /* edx: bytes to zero, rdi: dest, eax:zero */

.Lzero_rest:

        movq %rdx,%rcx

.Le_byte:

        xorl %eax,%eax

.Le5:   rep

        stosb

        /* when there is another exception while zeroing the rest just return */

.Le_zero:

        movq %rdx,%rax

        jmp .Lende