1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * allocation tagging
4 */
5#ifndef _LINUX_ALLOC_TAG_H
6#define _LINUX_ALLOC_TAG_H
7
8#include <linux/bug.h>
9#include <linux/codetag.h>
10#include <linux/container_of.h>
11#include <linux/preempt.h>
12#include <asm/percpu.h>
13#include <linux/cpumask.h>
14#include <linux/smp.h>
15#include <linux/static_key.h>
16#include <linux/irqflags.h>
17
18struct alloc_tag_counters {
19	u64 bytes;
20	u64 calls;
21};
22
23/*
24 * An instance of this structure is created in a special ELF section at every
25 * allocation callsite. At runtime, the special section is treated as
26 * an array of these. Embedded codetag utilizes codetag framework.
27 */
28struct alloc_tag {
29	struct codetag			ct;
30	struct alloc_tag_counters __percpu	*counters;
31} __aligned(8);
32
33#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
34
35#define CODETAG_EMPTY	((void *)1)
36
37static inline bool is_codetag_empty(union codetag_ref *ref)
38{
39	return ref->ct == CODETAG_EMPTY;
40}
41
42static inline void set_codetag_empty(union codetag_ref *ref)
43{
44	if (ref)
45		ref->ct = CODETAG_EMPTY;
46}
47
48#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
49
50static inline bool is_codetag_empty(union codetag_ref *ref) { return false; }
51static inline void set_codetag_empty(union codetag_ref *ref) {}
52
53#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
54
55#ifdef CONFIG_MEM_ALLOC_PROFILING
56
57struct codetag_bytes {
58	struct codetag *ct;
59	s64 bytes;
60};
61
62size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep);
63
64static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct)
65{
66	return container_of(ct, struct alloc_tag, ct);
67}
68
69#ifdef ARCH_NEEDS_WEAK_PER_CPU
70/*
71 * When percpu variables are required to be defined as weak, static percpu
72 * variables can't be used inside a function (see comments for DECLARE_PER_CPU_SECTION).
73 * Instead we will accound all module allocations to a single counter.
74 */
75DECLARE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
76
77#define DEFINE_ALLOC_TAG(_alloc_tag)						\
78	static struct alloc_tag _alloc_tag __used __aligned(8)			\
79	__section("alloc_tags") = {						\
80		.ct = CODE_TAG_INIT,						\
81		.counters = &_shared_alloc_tag };
82
83#else /* ARCH_NEEDS_WEAK_PER_CPU */
84
85#define DEFINE_ALLOC_TAG(_alloc_tag)						\
86	static DEFINE_PER_CPU(struct alloc_tag_counters, _alloc_tag_cntr);	\
87	static struct alloc_tag _alloc_tag __used __aligned(8)			\
88	__section("alloc_tags") = {						\
89		.ct = CODE_TAG_INIT,						\
90		.counters = &_alloc_tag_cntr };
91
92#endif /* ARCH_NEEDS_WEAK_PER_CPU */
93
94DECLARE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
95			mem_alloc_profiling_key);
96
97static inline bool mem_alloc_profiling_enabled(void)
98{
99	return static_branch_maybe(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
100				   &mem_alloc_profiling_key);
101}
102
103static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag)
104{
105	struct alloc_tag_counters v = { 0, 0 };
106	struct alloc_tag_counters *counter;
107	int cpu;
108
109	for_each_possible_cpu(cpu) {
110		counter = per_cpu_ptr(tag->counters, cpu);
111		v.bytes += counter->bytes;
112		v.calls += counter->calls;
113	}
114
115	return v;
116}
117
118#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
119static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag)
120{
121	WARN_ONCE(ref && ref->ct,
122		  "alloc_tag was not cleared (got tag for %s:%u)\n",
123		  ref->ct->filename, ref->ct->lineno);
124
125	WARN_ONCE(!tag, "current->alloc_tag not set");
126}
127
128static inline void alloc_tag_sub_check(union codetag_ref *ref)
129{
130	WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n");
131}
132#else
133static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {}
134static inline void alloc_tag_sub_check(union codetag_ref *ref) {}
135#endif
136
137/* Caller should verify both ref and tag to be valid */
138static inline void __alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)
139{
140	ref->ct = &tag->ct;
141	/*
142	 * We need in increment the call counter every time we have a new
143	 * allocation or when we split a large allocation into smaller ones.
144	 * Each new reference for every sub-allocation needs to increment call
145	 * counter because when we free each part the counter will be decremented.
146	 */
147	this_cpu_inc(tag->counters->calls);
148}
149
150static inline void alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)
151{
152	alloc_tag_add_check(ref, tag);
153	if (!ref || !tag)
154		return;
155
156	__alloc_tag_ref_set(ref, tag);
157}
158
159static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes)
160{
161	alloc_tag_add_check(ref, tag);
162	if (!ref || !tag)
163		return;
164
165	__alloc_tag_ref_set(ref, tag);
166	this_cpu_add(tag->counters->bytes, bytes);
167}
168
169static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes)
170{
171	struct alloc_tag *tag;
172
173	alloc_tag_sub_check(ref);
174	if (!ref || !ref->ct)
175		return;
176
177	if (is_codetag_empty(ref)) {
178		ref->ct = NULL;
179		return;
180	}
181
182	tag = ct_to_alloc_tag(ref->ct);
183
184	this_cpu_sub(tag->counters->bytes, bytes);
185	this_cpu_dec(tag->counters->calls);
186
187	ref->ct = NULL;
188}
189
190#define alloc_tag_record(p)	((p) = current->alloc_tag)
191
192#else /* CONFIG_MEM_ALLOC_PROFILING */
193
194#define DEFINE_ALLOC_TAG(_alloc_tag)
195static inline bool mem_alloc_profiling_enabled(void) { return false; }
196static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag,
197				 size_t bytes) {}
198static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {}
199#define alloc_tag_record(p)	do {} while (0)
200
201#endif /* CONFIG_MEM_ALLOC_PROFILING */
202
203#define alloc_hooks_tag(_tag, _do_alloc)				\
204({									\
205	struct alloc_tag * __maybe_unused _old = alloc_tag_save(_tag);	\
206	typeof(_do_alloc) _res = _do_alloc;				\
207	alloc_tag_restore(_tag, _old);					\
208	_res;								\
209})
210
211#define alloc_hooks(_do_alloc)						\
212({									\
213	DEFINE_ALLOC_TAG(_alloc_tag);					\
214	alloc_hooks_tag(&_alloc_tag, _do_alloc);			\
215})
216
217#endif /* _LINUX_ALLOC_TAG_H */
218