Commit a82eee74 authored by Toshi Kani's avatar Toshi Kani Committed by Ingo Molnar

x86/uaccess/64: Handle the caching of 4-byte nocache copies properly in __copy_user_nocache()

Data corruption issues were observed in tests which initiated
a system crash/reset while accessing BTT devices.  This problem
is reproducible.

The BTT driver calls pmem_rw_bytes() to update data in pmem
devices.  This interface calls __copy_user_nocache(), which
uses non-temporal stores so that the stores to pmem are
persistent.

__copy_user_nocache() uses non-temporal stores when a request
size is 8 bytes or larger (and is aligned by 8 bytes).  The
BTT driver updates the BTT map table, which entry size is
4 bytes.  Therefore, updates to the map table entries remain
cached, and are not written to pmem after a crash.

Change __copy_user_nocache() to use non-temporal store when
a request size is 4 bytes.  The change extends the current
byte-copy path for a less-than-8-bytes request, and does not
add any overhead to the regular path.
Reported-and-tested-by: default avatarMicah Parrish <micah.parrish@hpe.com>
Reported-and-tested-by: default avatarBrian Boylston <brian.boylston@hpe.com>
Signed-off-by: default avatarToshi Kani <toshi.kani@hpe.com>
Cc: <stable@vger.kernel.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Borislav Petkov <bp@suse.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Luis R. Rodriguez <mcgrof@suse.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Toshi Kani <toshi.kani@hp.com>
Cc: Vishal Verma <vishal.l.verma@intel.com>
Cc: linux-nvdimm@lists.01.org
Link: http://lkml.kernel.org/r/1455225857-12039-3-git-send-email-toshi.kani@hpe.com
[ Small readability edits. ]
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent ee9737c9
...@@ -237,13 +237,14 @@ ENDPROC(copy_user_enhanced_fast_string) ...@@ -237,13 +237,14 @@ ENDPROC(copy_user_enhanced_fast_string)
* Note: Cached memory copy is used when destination or size is not * Note: Cached memory copy is used when destination or size is not
* naturally aligned. That is: * naturally aligned. That is:
* - Require 8-byte alignment when size is 8 bytes or larger. * - Require 8-byte alignment when size is 8 bytes or larger.
* - Require 4-byte alignment when size is 4 bytes.
*/ */
ENTRY(__copy_user_nocache) ENTRY(__copy_user_nocache)
ASM_STAC ASM_STAC
/* If size is less than 8 bytes, go to byte copy */ /* If size is less than 8 bytes, go to 4-byte copy */
cmpl $8,%edx cmpl $8,%edx
jb .L_1b_cache_copy_entry jb .L_4b_nocache_copy_entry
/* If destination is not 8-byte aligned, "cache" copy to align it */ /* If destination is not 8-byte aligned, "cache" copy to align it */
ALIGN_DESTINATION ALIGN_DESTINATION
...@@ -282,7 +283,7 @@ ENTRY(__copy_user_nocache) ...@@ -282,7 +283,7 @@ ENTRY(__copy_user_nocache)
movl %edx,%ecx movl %edx,%ecx
andl $7,%edx andl $7,%edx
shrl $3,%ecx shrl $3,%ecx
jz .L_1b_cache_copy_entry /* jump if count is 0 */ jz .L_4b_nocache_copy_entry /* jump if count is 0 */
/* Perform 8-byte nocache loop-copy */ /* Perform 8-byte nocache loop-copy */
.L_8b_nocache_copy_loop: .L_8b_nocache_copy_loop:
...@@ -294,11 +295,33 @@ ENTRY(__copy_user_nocache) ...@@ -294,11 +295,33 @@ ENTRY(__copy_user_nocache)
jnz .L_8b_nocache_copy_loop jnz .L_8b_nocache_copy_loop
/* If no byte left, we're done */ /* If no byte left, we're done */
.L_1b_cache_copy_entry: .L_4b_nocache_copy_entry:
andl %edx,%edx
jz .L_finish_copy
/* If destination is not 4-byte aligned, go to byte copy: */
movl %edi,%ecx
andl $3,%ecx
jnz .L_1b_cache_copy_entry
/* Set 4-byte copy count (1 or 0) and remainder */
movl %edx,%ecx
andl $3,%edx
shrl $2,%ecx
jz .L_1b_cache_copy_entry /* jump if count is 0 */
/* Perform 4-byte nocache copy: */
30: movl (%rsi),%r8d
31: movnti %r8d,(%rdi)
leaq 4(%rsi),%rsi
leaq 4(%rdi),%rdi
/* If no bytes left, we're done: */
andl %edx,%edx andl %edx,%edx
jz .L_finish_copy jz .L_finish_copy
/* Perform byte "cache" loop-copy for the remainder */ /* Perform byte "cache" loop-copy for the remainder */
.L_1b_cache_copy_entry:
movl %edx,%ecx movl %edx,%ecx
.L_1b_cache_copy_loop: .L_1b_cache_copy_loop:
40: movb (%rsi),%al 40: movb (%rsi),%al
...@@ -323,6 +346,9 @@ ENTRY(__copy_user_nocache) ...@@ -323,6 +346,9 @@ ENTRY(__copy_user_nocache)
.L_fixup_8b_copy: .L_fixup_8b_copy:
lea (%rdx,%rcx,8),%rdx lea (%rdx,%rcx,8),%rdx
jmp .L_fixup_handle_tail jmp .L_fixup_handle_tail
.L_fixup_4b_copy:
lea (%rdx,%rcx,4),%rdx
jmp .L_fixup_handle_tail
.L_fixup_1b_copy: .L_fixup_1b_copy:
movl %ecx,%edx movl %ecx,%edx
.L_fixup_handle_tail: .L_fixup_handle_tail:
...@@ -348,6 +374,8 @@ ENTRY(__copy_user_nocache) ...@@ -348,6 +374,8 @@ ENTRY(__copy_user_nocache)
_ASM_EXTABLE(16b,.L_fixup_4x8b_copy) _ASM_EXTABLE(16b,.L_fixup_4x8b_copy)
_ASM_EXTABLE(20b,.L_fixup_8b_copy) _ASM_EXTABLE(20b,.L_fixup_8b_copy)
_ASM_EXTABLE(21b,.L_fixup_8b_copy) _ASM_EXTABLE(21b,.L_fixup_8b_copy)
_ASM_EXTABLE(30b,.L_fixup_4b_copy)
_ASM_EXTABLE(31b,.L_fixup_4b_copy)
_ASM_EXTABLE(40b,.L_fixup_1b_copy) _ASM_EXTABLE(40b,.L_fixup_1b_copy)
_ASM_EXTABLE(41b,.L_fixup_1b_copy) _ASM_EXTABLE(41b,.L_fixup_1b_copy)
ENDPROC(__copy_user_nocache) ENDPROC(__copy_user_nocache)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment