1// SPDX-License-Identifier: GPL-2.0-only OR MIT
2/* Copyright (c) 2023 Imagination Technologies Ltd. */
3
4#include <uapi/drm/pvr_drm.h>
5
6#include <drm/drm_syncobj.h>
7#include <drm/gpu_scheduler.h>
8#include <linux/xarray.h>
9#include <linux/dma-fence-unwrap.h>
10
11#include "pvr_device.h"
12#include "pvr_queue.h"
13#include "pvr_sync.h"
14
15static int
16pvr_check_sync_op(const struct drm_pvr_sync_op *sync_op)
17{
18	u8 handle_type;
19
20	if (sync_op->flags & ~DRM_PVR_SYNC_OP_FLAGS_MASK)
21		return -EINVAL;
22
23	handle_type = sync_op->flags & DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_MASK;
24	if (handle_type != DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_SYNCOBJ &&
25	    handle_type != DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_TIMELINE_SYNCOBJ)
26		return -EINVAL;
27
28	if (handle_type == DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_SYNCOBJ &&
29	    sync_op->value != 0)
30		return -EINVAL;
31
32	return 0;
33}
34
35static void
36pvr_sync_signal_free(struct pvr_sync_signal *sig_sync)
37{
38	if (!sig_sync)
39		return;
40
41	drm_syncobj_put(sig_sync->syncobj);
42	dma_fence_chain_free(sig_sync->chain);
43	dma_fence_put(sig_sync->fence);
44	kfree(sig_sync);
45}
46
47void
48pvr_sync_signal_array_cleanup(struct xarray *array)
49{
50	struct pvr_sync_signal *sig_sync;
51	unsigned long i;
52
53	xa_for_each(array, i, sig_sync)
54		pvr_sync_signal_free(sig_sync);
55
56	xa_destroy(array);
57}
58
59static struct pvr_sync_signal *
60pvr_sync_signal_array_add(struct xarray *array, struct drm_file *file, u32 handle, u64 point)
61{
62	struct pvr_sync_signal *sig_sync;
63	struct dma_fence *cur_fence;
64	int err;
65	u32 id;
66
67	sig_sync = kzalloc(sizeof(*sig_sync), GFP_KERNEL);
68	if (!sig_sync)
69		return ERR_PTR(-ENOMEM);
70
71	sig_sync->handle = handle;
72	sig_sync->point = point;
73
74	if (point > 0) {
75		sig_sync->chain = dma_fence_chain_alloc();
76		if (!sig_sync->chain) {
77			err = -ENOMEM;
78			goto err_free_sig_sync;
79		}
80	}
81
82	sig_sync->syncobj = drm_syncobj_find(file, handle);
83	if (!sig_sync->syncobj) {
84		err = -EINVAL;
85		goto err_free_sig_sync;
86	}
87
88	/* Retrieve the current fence attached to that point. It's
89	 * perfectly fine to get a NULL fence here, it just means there's
90	 * no fence attached to that point yet.
91	 */
92	if (!drm_syncobj_find_fence(file, handle, point, 0, &cur_fence))
93		sig_sync->fence = cur_fence;
94
95	err = xa_alloc(array, &id, sig_sync, xa_limit_32b, GFP_KERNEL);
96	if (err)
97		goto err_free_sig_sync;
98
99	return sig_sync;
100
101err_free_sig_sync:
102	pvr_sync_signal_free(sig_sync);
103	return ERR_PTR(err);
104}
105
106static struct pvr_sync_signal *
107pvr_sync_signal_array_search(struct xarray *array, u32 handle, u64 point)
108{
109	struct pvr_sync_signal *sig_sync;
110	unsigned long i;
111
112	xa_for_each(array, i, sig_sync) {
113		if (handle == sig_sync->handle && point == sig_sync->point)
114			return sig_sync;
115	}
116
117	return NULL;
118}
119
120static struct pvr_sync_signal *
121pvr_sync_signal_array_get(struct xarray *array, struct drm_file *file, u32 handle, u64 point)
122{
123	struct pvr_sync_signal *sig_sync;
124
125	sig_sync = pvr_sync_signal_array_search(array, handle, point);
126	if (sig_sync)
127		return sig_sync;
128
129	return pvr_sync_signal_array_add(array, file, handle, point);
130}
131
132int
133pvr_sync_signal_array_collect_ops(struct xarray *array,
134				  struct drm_file *file,
135				  u32 sync_op_count,
136				  const struct drm_pvr_sync_op *sync_ops)
137{
138	for (u32 i = 0; i < sync_op_count; i++) {
139		struct pvr_sync_signal *sig_sync;
140		int ret;
141
142		if (!(sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL))
143			continue;
144
145		ret = pvr_check_sync_op(&sync_ops[i]);
146		if (ret)
147			return ret;
148
149		sig_sync = pvr_sync_signal_array_get(array, file,
150						     sync_ops[i].handle,
151						     sync_ops[i].value);
152		if (IS_ERR(sig_sync))
153			return PTR_ERR(sig_sync);
154	}
155
156	return 0;
157}
158
159int
160pvr_sync_signal_array_update_fences(struct xarray *array,
161				    u32 sync_op_count,
162				    const struct drm_pvr_sync_op *sync_ops,
163				    struct dma_fence *done_fence)
164{
165	for (u32 i = 0; i < sync_op_count; i++) {
166		struct dma_fence *old_fence;
167		struct pvr_sync_signal *sig_sync;
168
169		if (!(sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL))
170			continue;
171
172		sig_sync = pvr_sync_signal_array_search(array, sync_ops[i].handle,
173							sync_ops[i].value);
174		if (WARN_ON(!sig_sync))
175			return -EINVAL;
176
177		old_fence = sig_sync->fence;
178		sig_sync->fence = dma_fence_get(done_fence);
179		dma_fence_put(old_fence);
180
181		if (WARN_ON(!sig_sync->fence))
182			return -EINVAL;
183	}
184
185	return 0;
186}
187
188void
189pvr_sync_signal_array_push_fences(struct xarray *array)
190{
191	struct pvr_sync_signal *sig_sync;
192	unsigned long i;
193
194	xa_for_each(array, i, sig_sync) {
195		if (sig_sync->chain) {
196			drm_syncobj_add_point(sig_sync->syncobj, sig_sync->chain,
197					      sig_sync->fence, sig_sync->point);
198			sig_sync->chain = NULL;
199		} else {
200			drm_syncobj_replace_fence(sig_sync->syncobj, sig_sync->fence);
201		}
202	}
203}
204
205static int
206pvr_sync_add_dep_to_job(struct drm_sched_job *job, struct dma_fence *f)
207{
208	struct dma_fence_unwrap iter;
209	u32 native_fence_count = 0;
210	struct dma_fence *uf;
211	int err = 0;
212
213	dma_fence_unwrap_for_each(uf, &iter, f) {
214		if (pvr_queue_fence_is_ufo_backed(uf))
215			native_fence_count++;
216	}
217
218	/* No need to unwrap the fence if it's fully non-native. */
219	if (!native_fence_count)
220		return drm_sched_job_add_dependency(job, f);
221
222	dma_fence_unwrap_for_each(uf, &iter, f) {
223		/* There's no dma_fence_unwrap_stop() helper cleaning up the refs
224		 * owned by dma_fence_unwrap(), so let's just iterate over all
225		 * entries without doing anything when something failed.
226		 */
227		if (err)
228			continue;
229
230		if (pvr_queue_fence_is_ufo_backed(uf)) {
231			struct drm_sched_fence *s_fence = to_drm_sched_fence(uf);
232
233			/* If this is a native dependency, we wait for the scheduled fence,
234			 * and we will let pvr_queue_run_job() issue FW waits.
235			 */
236			err = drm_sched_job_add_dependency(job,
237							   dma_fence_get(&s_fence->scheduled));
238		} else {
239			err = drm_sched_job_add_dependency(job, dma_fence_get(uf));
240		}
241	}
242
243	dma_fence_put(f);
244	return err;
245}
246
247int
248pvr_sync_add_deps_to_job(struct pvr_file *pvr_file, struct drm_sched_job *job,
249			 u32 sync_op_count,
250			 const struct drm_pvr_sync_op *sync_ops,
251			 struct xarray *signal_array)
252{
253	int err = 0;
254
255	if (!sync_op_count)
256		return 0;
257
258	for (u32 i = 0; i < sync_op_count; i++) {
259		struct pvr_sync_signal *sig_sync;
260		struct dma_fence *fence;
261
262		if (sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL)
263			continue;
264
265		err = pvr_check_sync_op(&sync_ops[i]);
266		if (err)
267			return err;
268
269		sig_sync = pvr_sync_signal_array_search(signal_array, sync_ops[i].handle,
270							sync_ops[i].value);
271		if (sig_sync) {
272			if (WARN_ON(!sig_sync->fence))
273				return -EINVAL;
274
275			fence = dma_fence_get(sig_sync->fence);
276		} else {
277			err = drm_syncobj_find_fence(from_pvr_file(pvr_file), sync_ops[i].handle,
278						     sync_ops[i].value, 0, &fence);
279			if (err)
280				return err;
281		}
282
283		err = pvr_sync_add_dep_to_job(job, fence);
284		if (err)
285			return err;
286	}
287
288	return 0;
289}
290