1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2015 Imagination Technologies Ltd
4 * Author: Qais Yousef <qais.yousef@imgtec.com>
5 *
6 * This file contains driver APIs to the IPI subsystem.
7 */
8
9#define pr_fmt(fmt) "genirq/ipi: " fmt
10
11#include <linux/irqdomain.h>
12#include <linux/irq.h>
13
14/**
15 * irq_reserve_ipi() - Setup an IPI to destination cpumask
16 * @domain:	IPI domain
17 * @dest:	cpumask of CPUs which can receive the IPI
18 *
19 * Allocate a virq that can be used to send IPI to any CPU in dest mask.
20 *
21 * Return: Linux IRQ number on success or error code on failure
22 */
23int irq_reserve_ipi(struct irq_domain *domain,
24			     const struct cpumask *dest)
25{
26	unsigned int nr_irqs, offset;
27	struct irq_data *data;
28	int virq, i;
29
30	if (!domain ||!irq_domain_is_ipi(domain)) {
31		pr_warn("Reservation on a non IPI domain\n");
32		return -EINVAL;
33	}
34
35	if (!cpumask_subset(dest, cpu_possible_mask)) {
36		pr_warn("Reservation is not in possible_cpu_mask\n");
37		return -EINVAL;
38	}
39
40	nr_irqs = cpumask_weight(dest);
41	if (!nr_irqs) {
42		pr_warn("Reservation for empty destination mask\n");
43		return -EINVAL;
44	}
45
46	if (irq_domain_is_ipi_single(domain)) {
47		/*
48		 * If the underlying implementation uses a single HW irq on
49		 * all cpus then we only need a single Linux irq number for
50		 * it. We have no restrictions vs. the destination mask. The
51		 * underlying implementation can deal with holes nicely.
52		 */
53		nr_irqs = 1;
54		offset = 0;
55	} else {
56		unsigned int next;
57
58		/*
59		 * The IPI requires a separate HW irq on each CPU. We require
60		 * that the destination mask is consecutive. If an
61		 * implementation needs to support holes, it can reserve
62		 * several IPI ranges.
63		 */
64		offset = cpumask_first(dest);
65		/*
66		 * Find a hole and if found look for another set bit after the
67		 * hole. For now we don't support this scenario.
68		 */
69		next = cpumask_next_zero(offset, dest);
70		if (next < nr_cpu_ids)
71			next = cpumask_next(next, dest);
72		if (next < nr_cpu_ids) {
73			pr_warn("Destination mask has holes\n");
74			return -EINVAL;
75		}
76	}
77
78	virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE, NULL);
79	if (virq <= 0) {
80		pr_warn("Can't reserve IPI, failed to alloc descs\n");
81		return -ENOMEM;
82	}
83
84	virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE,
85				       (void *) dest, true, NULL);
86
87	if (virq <= 0) {
88		pr_warn("Can't reserve IPI, failed to alloc hw irqs\n");
89		goto free_descs;
90	}
91
92	for (i = 0; i < nr_irqs; i++) {
93		data = irq_get_irq_data(virq + i);
94		cpumask_copy(data->common->affinity, dest);
95		data->common->ipi_offset = offset;
96		irq_set_status_flags(virq + i, IRQ_NO_BALANCING);
97	}
98	return virq;
99
100free_descs:
101	irq_free_descs(virq, nr_irqs);
102	return -EBUSY;
103}
104
105/**
106 * irq_destroy_ipi() - unreserve an IPI that was previously allocated
107 * @irq:	Linux IRQ number to be destroyed
108 * @dest:	cpumask of CPUs which should have the IPI removed
109 *
110 * The IPIs allocated with irq_reserve_ipi() are returned to the system
111 * destroying all virqs associated with them.
112 *
113 * Return: %0 on success or error code on failure.
114 */
115int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest)
116{
117	struct irq_data *data = irq_get_irq_data(irq);
118	const struct cpumask *ipimask;
119	struct irq_domain *domain;
120	unsigned int nr_irqs;
121
122	if (!irq || !data)
123		return -EINVAL;
124
125	domain = data->domain;
126	if (WARN_ON(domain == NULL))
127		return -EINVAL;
128
129	if (!irq_domain_is_ipi(domain)) {
130		pr_warn("Trying to destroy a non IPI domain!\n");
131		return -EINVAL;
132	}
133
134	ipimask = irq_data_get_affinity_mask(data);
135	if (!ipimask || WARN_ON(!cpumask_subset(dest, ipimask)))
136		/*
137		 * Must be destroying a subset of CPUs to which this IPI
138		 * was set up to target
139		 */
140		return -EINVAL;
141
142	if (irq_domain_is_ipi_per_cpu(domain)) {
143		irq = irq + cpumask_first(dest) - data->common->ipi_offset;
144		nr_irqs = cpumask_weight(dest);
145	} else {
146		nr_irqs = 1;
147	}
148
149	irq_domain_free_irqs(irq, nr_irqs);
150	return 0;
151}
152
153/**
154 * ipi_get_hwirq - Get the hwirq associated with an IPI to a CPU
155 * @irq:	Linux IRQ number
156 * @cpu:	the target CPU
157 *
158 * When dealing with coprocessors IPI, we need to inform the coprocessor of
159 * the hwirq it needs to use to receive and send IPIs.
160 *
161 * Return: hwirq value on success or INVALID_HWIRQ on failure.
162 */
163irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu)
164{
165	struct irq_data *data = irq_get_irq_data(irq);
166	const struct cpumask *ipimask;
167
168	if (!data || cpu >= nr_cpu_ids)
169		return INVALID_HWIRQ;
170
171	ipimask = irq_data_get_affinity_mask(data);
172	if (!ipimask || !cpumask_test_cpu(cpu, ipimask))
173		return INVALID_HWIRQ;
174
175	/*
176	 * Get the real hardware irq number if the underlying implementation
177	 * uses a separate irq per cpu. If the underlying implementation uses
178	 * a single hardware irq for all cpus then the IPI send mechanism
179	 * needs to take care of the cpu destinations.
180	 */
181	if (irq_domain_is_ipi_per_cpu(data->domain))
182		data = irq_get_irq_data(irq + cpu - data->common->ipi_offset);
183
184	return data ? irqd_to_hwirq(data) : INVALID_HWIRQ;
185}
186EXPORT_SYMBOL_GPL(ipi_get_hwirq);
187
188static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data,
189			   const struct cpumask *dest, unsigned int cpu)
190{
191	const struct cpumask *ipimask;
192
193	if (!chip || !data)
194		return -EINVAL;
195
196	if (!chip->ipi_send_single && !chip->ipi_send_mask)
197		return -EINVAL;
198
199	if (cpu >= nr_cpu_ids)
200		return -EINVAL;
201
202	ipimask = irq_data_get_affinity_mask(data);
203	if (!ipimask)
204		return -EINVAL;
205
206	if (dest) {
207		if (!cpumask_subset(dest, ipimask))
208			return -EINVAL;
209	} else {
210		if (!cpumask_test_cpu(cpu, ipimask))
211			return -EINVAL;
212	}
213	return 0;
214}
215
216/**
217 * __ipi_send_single - send an IPI to a target Linux SMP CPU
218 * @desc:	pointer to irq_desc of the IRQ
219 * @cpu:	destination CPU, must in the destination mask passed to
220 *		irq_reserve_ipi()
221 *
222 * This function is for architecture or core code to speed up IPI sending. Not
223 * usable from driver code.
224 *
225 * Return: %0 on success or negative error number on failure.
226 */
227int __ipi_send_single(struct irq_desc *desc, unsigned int cpu)
228{
229	struct irq_data *data = irq_desc_get_irq_data(desc);
230	struct irq_chip *chip = irq_data_get_irq_chip(data);
231
232#ifdef DEBUG
233	/*
234	 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
235	 * Since the callers should be arch or core code which is generally
236	 * trusted, only check for errors when debugging.
237	 */
238	if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
239		return -EINVAL;
240#endif
241	if (!chip->ipi_send_single) {
242		chip->ipi_send_mask(data, cpumask_of(cpu));
243		return 0;
244	}
245
246	/* FIXME: Store this information in irqdata flags */
247	if (irq_domain_is_ipi_per_cpu(data->domain) &&
248	    cpu != data->common->ipi_offset) {
249		/* use the correct data for that cpu */
250		unsigned irq = data->irq + cpu - data->common->ipi_offset;
251
252		data = irq_get_irq_data(irq);
253	}
254	chip->ipi_send_single(data, cpu);
255	return 0;
256}
257
258/**
259 * __ipi_send_mask - send an IPI to target Linux SMP CPU(s)
260 * @desc:	pointer to irq_desc of the IRQ
261 * @dest:	dest CPU(s), must be a subset of the mask passed to
262 *		irq_reserve_ipi()
263 *
264 * This function is for architecture or core code to speed up IPI sending. Not
265 * usable from driver code.
266 *
267 * Return: %0 on success or negative error number on failure.
268 */
269int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest)
270{
271	struct irq_data *data = irq_desc_get_irq_data(desc);
272	struct irq_chip *chip = irq_data_get_irq_chip(data);
273	unsigned int cpu;
274
275#ifdef DEBUG
276	/*
277	 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
278	 * Since the callers should be arch or core code which is generally
279	 * trusted, only check for errors when debugging.
280	 */
281	if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
282		return -EINVAL;
283#endif
284	if (chip->ipi_send_mask) {
285		chip->ipi_send_mask(data, dest);
286		return 0;
287	}
288
289	if (irq_domain_is_ipi_per_cpu(data->domain)) {
290		unsigned int base = data->irq;
291
292		for_each_cpu(cpu, dest) {
293			unsigned irq = base + cpu - data->common->ipi_offset;
294
295			data = irq_get_irq_data(irq);
296			chip->ipi_send_single(data, cpu);
297		}
298	} else {
299		for_each_cpu(cpu, dest)
300			chip->ipi_send_single(data, cpu);
301	}
302	return 0;
303}
304
305/**
306 * ipi_send_single - Send an IPI to a single CPU
307 * @virq:	Linux IRQ number from irq_reserve_ipi()
308 * @cpu:	destination CPU, must in the destination mask passed to
309 *		irq_reserve_ipi()
310 *
311 * Return: %0 on success or negative error number on failure.
312 */
313int ipi_send_single(unsigned int virq, unsigned int cpu)
314{
315	struct irq_desc *desc = irq_to_desc(virq);
316	struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
317	struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
318
319	if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
320		return -EINVAL;
321
322	return __ipi_send_single(desc, cpu);
323}
324EXPORT_SYMBOL_GPL(ipi_send_single);
325
326/**
327 * ipi_send_mask - Send an IPI to target CPU(s)
328 * @virq:	Linux IRQ number from irq_reserve_ipi()
329 * @dest:	dest CPU(s), must be a subset of the mask passed to
330 *		irq_reserve_ipi()
331 *
332 * Return: %0 on success or negative error number on failure.
333 */
334int ipi_send_mask(unsigned int virq, const struct cpumask *dest)
335{
336	struct irq_desc *desc = irq_to_desc(virq);
337	struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
338	struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
339
340	if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
341		return -EINVAL;
342
343	return __ipi_send_mask(desc, dest);
344}
345EXPORT_SYMBOL_GPL(ipi_send_mask);
346