[SSI] openssi/kernel/arch/x86_64/lib copy_user.S, 1.1, 1.2 usercopy.c, 1.1, 1.2
John Hughes <[email protected]> Mon, 15 Feb 2010 16:48:45 +0000
| Newsgroups | gmane.linux.cluster.ssic.cvs |
|---|---|
| Message-ID | <[email protected]> |
Update of /cvsroot/ssic-linux/openssi/kernel/arch/x86_64/lib
In directory sfp-cvsdas-3.v30.ch3.sourceforge.com:/tmp/cvs-serv27464/arch/x86_64/lib
Modified Files:
Tag: OPENSSI-DEBIAN
copy_user.S usercopy.c
Log Message:
Back out incorrect x86_64 patch
Index: usercopy.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/arch/x86_64/lib/usercopy.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- usercopy.c 13 Jun 2003 14:51:32 -0000 1.1
+++ usercopy.c 15 Feb 2010 16:48:43 -0000 1.2
@@ -14,6 +14,7 @@
#define __do_strncpy_from_user(dst,src,count,res) \
do { \
long __d0, __d1, __d2; \
+ might_sleep(); \
__asm__ __volatile__( \
" testq %1,%1\n" \
" jz 2f\n" \
@@ -40,7 +41,7 @@
} while (0)
long
-__strncpy_from_user(char *dst, const char *src, long count)
+__strncpy_from_user(char *dst, const char __user *src, long count)
{
long res;
__do_strncpy_from_user(dst, src, count, res);
@@ -48,7 +49,7 @@
}
long
-strncpy_from_user(char *dst, const char *src, long count)
+strncpy_from_user(char *dst, const char __user *src, long count)
{
long res = -EFAULT;
if (access_ok(VERIFY_READ, src, 1))
@@ -60,15 +61,16 @@
* Zero Userspace
*/
-unsigned long __clear_user(void *addr, unsigned long size)
+unsigned long __clear_user(void __user *addr, unsigned long size)
{
long __d0;
+ might_sleep();
/* no memory constraint because it doesn't change any memory gcc knows
about */
asm volatile(
" testq %[size8],%[size8]\n"
" jz 4f\n"
- "0: movnti %[zero],(%[dst])\n"
+ "0: movq %[zero],(%[dst])\n"
" addq %[eight],%[dst]\n"
" decl %%ecx ; jnz 0b\n"
"4: movq %[size1],%%rcx\n"
@@ -77,7 +79,7 @@
"1: movb %b[zero],(%[dst])\n"
" incq %[dst]\n"
" decl %%ecx ; jnz 1b\n"
- "2: sfence\n"
+ "2:\n"
".section .fixup,\"ax\"\n"
"3: lea 0(%[size1],%[size8],8),%[size8]\n"
" jmp 2b\n"
@@ -88,13 +90,13 @@
" .quad 1b,2b\n"
".previous"
: [size8] "=c"(size), [dst] "=&D" (__d0)
- : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst] "(addr),
+ : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr),
[zero] "r" (0UL), [eight] "r" (8UL));
return size;
}
-unsigned long clear_user(void *to, unsigned long n)
+unsigned long clear_user(void __user *to, unsigned long n)
{
if (access_ok(VERIFY_WRITE, to, n))
return __clear_user(to, n);
@@ -107,22 +109,45 @@
* Return 0 on exception, a value greater than N if too long
*/
-long strnlen_user(const char *s, long n)
+long strnlen_user(const char __user *s, long n)
{
- unsigned long res = 0;
+ long res = 0;
char c;
if (!access_ok(VERIFY_READ, s, n))
return 0;
while (1) {
+ if (res>n)
+ return n+1;
+ if (__get_user(c, s))
+ return 0;
+ if (!c)
+ return res+1;
+ res++;
+ s++;
+ }
+}
+
+long strlen_user(const char __user *s)
+{
+ long res = 0;
+ char c;
+
+ for (;;) {
if (get_user(c, s))
return 0;
if (!c)
return res+1;
- if (res>n)
- return n+1;
res++;
s++;
}
}
+
+unsigned long copy_in_user(void __user *to, const void __user *from, unsigned len)
+{
+ if (access_ok(VERIFY_WRITE, to, len) && access_ok(VERIFY_READ, from, len)) {
+ return copy_user_generic((__force void *)to, (__force void *)from, len);
+ }
+ return len;
+}
Index: copy_user.S
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/arch/x86_64/lib/copy_user.S,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- copy_user.S 13 Jun 2003 14:51:32 -0000 1.1
+++ copy_user.S 15 Feb 2010 16:48:43 -0000 1.2
@@ -5,31 +5,49 @@
*/
#define FIX_ALIGNMENT 1
-
+
#include <asm/current.h>
#include <asm/offset.h>
+ #include <asm/thread_info.h>
+ #include <asm/cpufeature.h>
/* Standard copy_to_user with segment limit checking */
.globl copy_to_user
.p2align 4
copy_to_user:
- GET_CURRENT(%rax)
+ GET_THREAD_INFO(%rax)
movq %rdi,%rcx
addq %rdx,%rcx
jc bad_to_user
- cmpq tsk_addr_limit(%rax),%rcx
+ cmpq threadinfo_addr_limit(%rax),%rcx
jae bad_to_user
- jmp copy_user_generic
+2:
+ .byte 0xe9 /* 32bit jump */
+ .long .Lcug-1f
+1:
+
+ .section .altinstr_replacement,"ax"
+3: .byte 0xe9 /* replacement jmp with 8 bit immediate */
+ .long copy_user_generic_c-1b /* offset */
+ .previous
+ .section .altinstructions,"a"
+ .align 8
+ .quad 2b
+ .quad 3b
+ .byte X86_FEATURE_K8_C
+ .byte 5
+ .byte 5
+ .previous
/* Standard copy_from_user with segment limit checking */
.globl copy_from_user
.p2align 4
copy_from_user:
- GET_CURRENT(%rax)
+ GET_THREAD_INFO(%rax)
movq %rsi,%rcx
addq %rdx,%rcx
jc bad_from_user
- cmpq tsk_addr_limit(%rax),%rcx
+ cmpq threadinfo_addr_limit(%rax),%rcx
jae bad_from_user
/* FALL THROUGH to copy_user_generic */
@@ -45,6 +63,7 @@
ret
.previous
+
/*
* copy_user_generic - memory copy with exception handling.
*
@@ -54,14 +73,27 @@
* rdx count
*
* Output:
- * eax uncopied bytes or 0 if successfull.
+ * eax uncopied bytes or 0 if successful.
*/
.globl copy_user_generic
.p2align 4
copy_user_generic:
- /* Put the first cacheline into cache. This should handle
- the small movements in ioctls etc., but not penalize the bigger
- filesystem data copies too much. */
+ .byte 0x66,0x66,0x90 /* 5 byte nop for replacement jump */
+ .byte 0x66,0x90
+1:
+ .section .altinstr_replacement,"ax"
+2: .byte 0xe9 /* near jump with 32bit immediate */
+ .long copy_user_generic_c-1b /* offset */
+ .previous
+ .section .altinstructions,"a"
+ .align 8
+ .quad copy_user_generic
+ .quad 2b
+ .byte X86_FEATURE_K8_C
+ .byte 5
+ .byte 5
+ .previous
+.Lcug:
pushq %rbx
xorl %eax,%eax /*zero for the exception handler */
@@ -147,9 +179,9 @@
movl $8,%r9d
subl %ecx,%r9d
movl %r9d,%ecx
- subq %r9,%rdx
- jz .Lsmall_align
- js .Lsmall_align
+ cmpq %r9,%rdx
+ jz .Lhandle_7
+ js .Lhandle_7
.Lalign_1:
.Ls11: movb (%rsi),%bl
.Ld11: movb %bl,(%rdi)
@@ -157,10 +189,8 @@
incq %rdi
decl %ecx
jnz .Lalign_1
+ subq %r9,%rdx
jmp .Lafter_bad_alignment
-.Lsmall_align:
- addq %r9,%rdx
- jmp .Lhandle_7
#endif
/* table sorted by exception address */
@@ -187,8 +217,8 @@
.quad .Ls10,.Le_byte
.quad .Ld10,.Le_byte
#ifdef FIX_ALIGNMENT
- .quad .Ls11,.Le_byte
- .quad .Ld11,.Le_byte
+ .quad .Ls11,.Lzero_rest
+ .quad .Ld11,.Lzero_rest
#endif
.quad .Le5,.Le_zero
.previous
@@ -231,3 +261,34 @@
.Le_zero:
movq %rdx,%rax
jmp .Lende
+
+ /* C stepping K8 run faster using the string copy instructions.
+ This is also a lot simpler. Use them when possible.
+ Patch in jmps to this code instead of copying it fully
+ to avoid unwanted aliasing in the exception tables. */
+
+ /* rdi destination
+ * rsi source
+ * rdx count
+ *
+ * Output:
+ * eax uncopied bytes or 0 if successfull.
+ */
+copy_user_generic_c:
+ movl %edx,%ecx
+ shrl $3,%ecx
+ andl $7,%edx
+1: rep
+ movsq
+ movl %edx,%ecx
+2: rep
+ movsb
+4: movl %ecx,%eax
+ ret
+3: lea (%rdx,%rcx,8),%rax
+ ret
+
+ .section __ex_table,"a"
+ .quad 1b,3b
+ .quad 2b,4b
+ .previous
------------------------------------------------------------------------------
SOLARIS 10 is the OS for Data Centers - provides features such as DTrace,
Predictive Self Healing and Award Winning ZFS. Get Solaris 10 NOW
http://p.sf.net/sfu/solaris-dev2dev