1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_X86_UACCESS_64_H
3#define _ASM_X86_UACCESS_64_H
4
5/*
6 * User space memory access functions
7 */
8#include <linux/compiler.h>
9#include <linux/lockdep.h>
10#include <linux/kasan-checks.h>
11#include <asm/alternative.h>
12#include <asm/cpufeatures.h>
13#include <asm/page.h>
14#include <asm/percpu.h>
15
16#ifdef CONFIG_ADDRESS_MASKING
17/*
18 * Mask out tag bits from the address.
19 */
20static inline unsigned long __untagged_addr(unsigned long addr)
21{
22	asm (ALTERNATIVE("",
23			 "and " __percpu_arg([mask]) ", %[addr]", X86_FEATURE_LAM)
24	     : [addr] "+r" (addr)
25	     : [mask] "m" (__my_cpu_var(tlbstate_untag_mask)));
26
27	return addr;
28}
29
30#define untagged_addr(addr)	({					\
31	unsigned long __addr = (__force unsigned long)(addr);		\
32	(__force __typeof__(addr))__untagged_addr(__addr);		\
33})
34
35static inline unsigned long __untagged_addr_remote(struct mm_struct *mm,
36						   unsigned long addr)
37{
38	mmap_assert_locked(mm);
39	return addr & (mm)->context.untag_mask;
40}
41
42#define untagged_addr_remote(mm, addr)	({				\
43	unsigned long __addr = (__force unsigned long)(addr);		\
44	(__force __typeof__(addr))__untagged_addr_remote(mm, __addr);	\
45})
46
47#endif
48
49/*
50 * The virtual address space space is logically divided into a kernel
51 * half and a user half.  When cast to a signed type, user pointers
52 * are positive and kernel pointers are negative.
53 */
54#define valid_user_address(x) ((__force long)(x) >= 0)
55
56/*
57 * User pointers can have tag bits on x86-64.  This scheme tolerates
58 * arbitrary values in those bits rather then masking them off.
59 *
60 * Enforce two rules:
61 * 1. 'ptr' must be in the user half of the address space
62 * 2. 'ptr+size' must not overflow into kernel addresses
63 *
64 * Note that addresses around the sign change are not valid addresses,
65 * and will GP-fault even with LAM enabled if the sign bit is set (see
66 * "CR3.LAM_SUP" that can narrow the canonicality check if we ever
67 * enable it, but not remove it entirely).
68 *
69 * So the "overflow into kernel addresses" does not imply some sudden
70 * exact boundary at the sign bit, and we can allow a lot of slop on the
71 * size check.
72 *
73 * In fact, we could probably remove the size check entirely, since
74 * any kernel accesses will be in increasing address order starting
75 * at 'ptr', and even if the end might be in kernel space, we'll
76 * hit the GP faults for non-canonical accesses before we ever get
77 * there.
78 *
79 * That's a separate optimization, for now just handle the small
80 * constant case.
81 */
82static inline bool __access_ok(const void __user *ptr, unsigned long size)
83{
84	if (__builtin_constant_p(size <= PAGE_SIZE) && size <= PAGE_SIZE) {
85		return valid_user_address(ptr);
86	} else {
87		unsigned long sum = size + (__force unsigned long)ptr;
88
89		return valid_user_address(sum) && sum >= (__force unsigned long)ptr;
90	}
91}
92#define __access_ok __access_ok
93
94/*
95 * Copy To/From Userspace
96 */
97
98/* Handles exceptions in both to and from, but doesn't do access_ok */
99__must_check unsigned long
100rep_movs_alternative(void *to, const void *from, unsigned len);
101
102static __always_inline __must_check unsigned long
103copy_user_generic(void *to, const void *from, unsigned long len)
104{
105	stac();
106	/*
107	 * If CPU has FSRM feature, use 'rep movs'.
108	 * Otherwise, use rep_movs_alternative.
109	 */
110	asm volatile(
111		"1:\n\t"
112		ALTERNATIVE("rep movsb",
113			    "call rep_movs_alternative", ALT_NOT(X86_FEATURE_FSRM))
114		"2:\n"
115		_ASM_EXTABLE_UA(1b, 2b)
116		:"+c" (len), "+D" (to), "+S" (from), ASM_CALL_CONSTRAINT
117		: : "memory", "rax");
118	clac();
119	return len;
120}
121
122static __always_inline __must_check unsigned long
123raw_copy_from_user(void *dst, const void __user *src, unsigned long size)
124{
125	return copy_user_generic(dst, (__force void *)src, size);
126}
127
128static __always_inline __must_check unsigned long
129raw_copy_to_user(void __user *dst, const void *src, unsigned long size)
130{
131	return copy_user_generic((__force void *)dst, src, size);
132}
133
134extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size);
135extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size);
136
137static inline int
138__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
139				  unsigned size)
140{
141	long ret;
142	kasan_check_write(dst, size);
143	stac();
144	ret = __copy_user_nocache(dst, src, size);
145	clac();
146	return ret;
147}
148
149static inline int
150__copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
151{
152	kasan_check_write(dst, size);
153	return __copy_user_flushcache(dst, src, size);
154}
155
156/*
157 * Zero Userspace.
158 */
159
160__must_check unsigned long
161rep_stos_alternative(void __user *addr, unsigned long len);
162
163static __always_inline __must_check unsigned long __clear_user(void __user *addr, unsigned long size)
164{
165	might_fault();
166	stac();
167
168	/*
169	 * No memory constraint because it doesn't change any memory gcc
170	 * knows about.
171	 */
172	asm volatile(
173		"1:\n\t"
174		ALTERNATIVE("rep stosb",
175			    "call rep_stos_alternative", ALT_NOT(X86_FEATURE_FSRS))
176		"2:\n"
177	       _ASM_EXTABLE_UA(1b, 2b)
178	       : "+c" (size), "+D" (addr), ASM_CALL_CONSTRAINT
179	       : "a" (0));
180
181	clac();
182
183	return size;
184}
185
186static __always_inline unsigned long clear_user(void __user *to, unsigned long n)
187{
188	if (__access_ok(to, n))
189		return __clear_user(to, n);
190	return n;
191}
192#endif /* _ASM_X86_UACCESS_64_H */
193