1239674Srwatson// SPDX-License-Identifier: GPL-2.0-only OR MIT 2245376Srwatson/* Copyright (c) 2023 Imagination Technologies Ltd. */ 3239674Srwatson 4239674Srwatson#include <uapi/drm/pvr_drm.h> 5239674Srwatson 6239674Srwatson#include <drm/drm_syncobj.h> 7239674Srwatson#include <drm/gpu_scheduler.h> 8239674Srwatson#include <linux/xarray.h> 9239674Srwatson#include <linux/dma-fence-unwrap.h> 10239674Srwatson 11239674Srwatson#include "pvr_device.h" 12239674Srwatson#include "pvr_queue.h" 13239674Srwatson#include "pvr_sync.h" 14239674Srwatson 15239674Srwatsonstatic int 16239674Srwatsonpvr_check_sync_op(const struct drm_pvr_sync_op *sync_op) 17239674Srwatson{ 18239674Srwatson u8 handle_type; 19239674Srwatson 20239674Srwatson if (sync_op->flags & ~DRM_PVR_SYNC_OP_FLAGS_MASK) 21239674Srwatson return -EINVAL; 22239674Srwatson 23239674Srwatson handle_type = sync_op->flags & DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_MASK; 24239674Srwatson if (handle_type != DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_SYNCOBJ && 25239674Srwatson handle_type != DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_TIMELINE_SYNCOBJ) 26239674Srwatson return -EINVAL; 27239674Srwatson 28239674Srwatson if (handle_type == DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_SYNCOBJ && 29239674Srwatson sync_op->value != 0) 30239674Srwatson return -EINVAL; 31239674Srwatson 32239674Srwatson return 0; 33239674Srwatson} 34239674Srwatson 35239674Srwatsonstatic void 36239674Srwatsonpvr_sync_signal_free(struct pvr_sync_signal *sig_sync) 37239674Srwatson{ 38239674Srwatson if (!sig_sync) 39239674Srwatson return; 40239674Srwatson 41239674Srwatson drm_syncobj_put(sig_sync->syncobj); 42239674Srwatson dma_fence_chain_free(sig_sync->chain); 43239674Srwatson dma_fence_put(sig_sync->fence); 44239674Srwatson kfree(sig_sync); 45239674Srwatson} 46239674Srwatson 47239674Srwatsonvoid 48239674Srwatsonpvr_sync_signal_array_cleanup(struct xarray *array) 49239674Srwatson{ 50239674Srwatson struct pvr_sync_signal *sig_sync; 51239674Srwatson unsigned long i; 52239674Srwatson 53239674Srwatson xa_for_each(array, i, sig_sync) 54239674Srwatson pvr_sync_signal_free(sig_sync); 55239674Srwatson 56239674Srwatson xa_destroy(array); 57239674Srwatson} 58239674Srwatson 59239674Srwatsonstatic struct pvr_sync_signal * 60239674Srwatsonpvr_sync_signal_array_add(struct xarray *array, struct drm_file *file, u32 handle, u64 point) 61239674Srwatson{ 62239674Srwatson struct pvr_sync_signal *sig_sync; 63245380Srwatson struct dma_fence *cur_fence; 64245380Srwatson int err; 65239674Srwatson u32 id; 66239674Srwatson 67239674Srwatson sig_sync = kzalloc(sizeof(*sig_sync), GFP_KERNEL); 68239674Srwatson if (!sig_sync) 69239674Srwatson return ERR_PTR(-ENOMEM); 70239674Srwatson 71239674Srwatson sig_sync->handle = handle; 72239674Srwatson sig_sync->point = point; 73239674Srwatson 74239674Srwatson if (point > 0) { 75239674Srwatson sig_sync->chain = dma_fence_chain_alloc(); 76239674Srwatson if (!sig_sync->chain) { 77239674Srwatson err = -ENOMEM; 78239674Srwatson goto err_free_sig_sync; 79239674Srwatson } 80239674Srwatson } 81239674Srwatson 82239674Srwatson sig_sync->syncobj = drm_syncobj_find(file, handle); 83239674Srwatson if (!sig_sync->syncobj) { 84239674Srwatson err = -EINVAL; 85239674Srwatson goto err_free_sig_sync; 86239674Srwatson } 87239674Srwatson 88239674Srwatson /* Retrieve the current fence attached to that point. It's 89239674Srwatson * perfectly fine to get a NULL fence here, it just means there's 90239674Srwatson * no fence attached to that point yet. 91239674Srwatson */ 92239674Srwatson if (!drm_syncobj_find_fence(file, handle, point, 0, &cur_fence)) 93239674Srwatson sig_sync->fence = cur_fence; 94239674Srwatson 95239674Srwatson err = xa_alloc(array, &id, sig_sync, xa_limit_32b, GFP_KERNEL); 96239674Srwatson if (err) 97239674Srwatson goto err_free_sig_sync; 98239674Srwatson 99239674Srwatson return sig_sync; 100239674Srwatson 101239674Srwatsonerr_free_sig_sync: 102239674Srwatson pvr_sync_signal_free(sig_sync); 103239674Srwatson return ERR_PTR(err); 104239674Srwatson} 105239674Srwatson 106239674Srwatsonstatic struct pvr_sync_signal * 107239674Srwatsonpvr_sync_signal_array_search(struct xarray *array, u32 handle, u64 point) 108239674Srwatson{ 109239674Srwatson struct pvr_sync_signal *sig_sync; 110239674Srwatson unsigned long i; 111239674Srwatson 112239674Srwatson xa_for_each(array, i, sig_sync) { 113239674Srwatson if (handle == sig_sync->handle && point == sig_sync->point) 114239674Srwatson return sig_sync; 115239674Srwatson } 116239674Srwatson 117239674Srwatson return NULL; 118239674Srwatson} 119239674Srwatson 120239674Srwatsonstatic struct pvr_sync_signal * 121239674Srwatsonpvr_sync_signal_array_get(struct xarray *array, struct drm_file *file, u32 handle, u64 point) 122239674Srwatson{ 123239674Srwatson struct pvr_sync_signal *sig_sync; 124239674Srwatson 125239674Srwatson sig_sync = pvr_sync_signal_array_search(array, handle, point); 126239674Srwatson if (sig_sync) 127239674Srwatson return sig_sync; 128239674Srwatson 129239674Srwatson return pvr_sync_signal_array_add(array, file, handle, point); 130239674Srwatson} 131239674Srwatson 132239674Srwatsonint 133239674Srwatsonpvr_sync_signal_array_collect_ops(struct xarray *array, 134239674Srwatson struct drm_file *file, 135239674Srwatson u32 sync_op_count, 136239674Srwatson const struct drm_pvr_sync_op *sync_ops) 137239674Srwatson{ 138239674Srwatson for (u32 i = 0; i < sync_op_count; i++) { 139239674Srwatson struct pvr_sync_signal *sig_sync; 140239674Srwatson int ret; 141239674Srwatson 142239674Srwatson if (!(sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL)) 143239674Srwatson continue; 144239674Srwatson 145239674Srwatson ret = pvr_check_sync_op(&sync_ops[i]); 146239674Srwatson if (ret) 147239674Srwatson return ret; 148239674Srwatson 149239674Srwatson sig_sync = pvr_sync_signal_array_get(array, file, 150239674Srwatson sync_ops[i].handle, 151239674Srwatson sync_ops[i].value); 152239674Srwatson if (IS_ERR(sig_sync)) 153239674Srwatson return PTR_ERR(sig_sync); 154239674Srwatson } 155239674Srwatson 156239674Srwatson return 0; 157239674Srwatson} 158239674Srwatson 159239674Srwatsonint 160239674Srwatsonpvr_sync_signal_array_update_fences(struct xarray *array, 161239674Srwatson u32 sync_op_count, 162239674Srwatson const struct drm_pvr_sync_op *sync_ops, 163239674Srwatson struct dma_fence *done_fence) 164239674Srwatson{ 165239674Srwatson for (u32 i = 0; i < sync_op_count; i++) { 166239674Srwatson struct dma_fence *old_fence; 167239674Srwatson struct pvr_sync_signal *sig_sync; 168239674Srwatson 169239674Srwatson if (!(sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL)) 170239674Srwatson continue; 171239674Srwatson 172239674Srwatson sig_sync = pvr_sync_signal_array_search(array, sync_ops[i].handle, 173239674Srwatson sync_ops[i].value); 174239674Srwatson if (WARN_ON(!sig_sync)) 175239674Srwatson return -EINVAL; 176239674Srwatson 177239674Srwatson old_fence = sig_sync->fence; 178239674Srwatson sig_sync->fence = dma_fence_get(done_fence); 179239674Srwatson dma_fence_put(old_fence); 180239674Srwatson 181239674Srwatson if (WARN_ON(!sig_sync->fence)) 182239674Srwatson return -EINVAL; 183239674Srwatson } 184239674Srwatson 185239674Srwatson return 0; 186239674Srwatson} 187239674Srwatson 188239674Srwatsonvoid 189239674Srwatsonpvr_sync_signal_array_push_fences(struct xarray *array) 190239674Srwatson{ 191239674Srwatson struct pvr_sync_signal *sig_sync; 192239674Srwatson unsigned long i; 193239674Srwatson 194239674Srwatson xa_for_each(array, i, sig_sync) { 195239674Srwatson if (sig_sync->chain) { 196239674Srwatson drm_syncobj_add_point(sig_sync->syncobj, sig_sync->chain, 197239674Srwatson sig_sync->fence, sig_sync->point); 198239674Srwatson sig_sync->chain = NULL; 199239674Srwatson } else { 200239674Srwatson drm_syncobj_replace_fence(sig_sync->syncobj, sig_sync->fence); 201239674Srwatson } 202239674Srwatson } 203239674Srwatson} 204239674Srwatson 205239674Srwatsonstatic int 206239674Srwatsonpvr_sync_add_dep_to_job(struct drm_sched_job *job, struct dma_fence *f) 207239674Srwatson{ 208239674Srwatson struct dma_fence_unwrap iter; 209239674Srwatson u32 native_fence_count = 0; 210239674Srwatson struct dma_fence *uf; 211239674Srwatson int err = 0; 212239674Srwatson 213239674Srwatson dma_fence_unwrap_for_each(uf, &iter, f) { 214239674Srwatson if (pvr_queue_fence_is_ufo_backed(uf)) 215239674Srwatson native_fence_count++; 216239674Srwatson } 217239674Srwatson 218239674Srwatson /* No need to unwrap the fence if it's fully non-native. */ 219239674Srwatson if (!native_fence_count) 220239674Srwatson return drm_sched_job_add_dependency(job, f); 221239674Srwatson 222239674Srwatson dma_fence_unwrap_for_each(uf, &iter, f) { 223239674Srwatson /* There's no dma_fence_unwrap_stop() helper cleaning up the refs 224239674Srwatson * owned by dma_fence_unwrap(), so let's just iterate over all 225239674Srwatson * entries without doing anything when something failed. 226239674Srwatson */ 227239674Srwatson if (err) 228239674Srwatson continue; 229239674Srwatson 230239674Srwatson if (pvr_queue_fence_is_ufo_backed(uf)) { 231239674Srwatson struct drm_sched_fence *s_fence = to_drm_sched_fence(uf); 232239674Srwatson 233239674Srwatson /* If this is a native dependency, we wait for the scheduled fence, 234239674Srwatson * and we will let pvr_queue_run_job() issue FW waits. 235239674Srwatson */ 236239674Srwatson err = drm_sched_job_add_dependency(job, 237239674Srwatson dma_fence_get(&s_fence->scheduled)); 238239674Srwatson } else { 239239674Srwatson err = drm_sched_job_add_dependency(job, dma_fence_get(uf)); 240239674Srwatson } 241239674Srwatson } 242239674Srwatson 243239674Srwatson dma_fence_put(f); 244239674Srwatson return err; 245239674Srwatson} 246239674Srwatson 247239674Srwatsonint 248239674Srwatsonpvr_sync_add_deps_to_job(struct pvr_file *pvr_file, struct drm_sched_job *job, 249239674Srwatson u32 sync_op_count, 250239674Srwatson const struct drm_pvr_sync_op *sync_ops, 251239674Srwatson struct xarray *signal_array) 252239674Srwatson{ 253239674Srwatson int err = 0; 254239674Srwatson 255239674Srwatson if (!sync_op_count) 256239674Srwatson return 0; 257239674Srwatson 258239674Srwatson for (u32 i = 0; i < sync_op_count; i++) { 259239674Srwatson struct pvr_sync_signal *sig_sync; 260239674Srwatson struct dma_fence *fence; 261239674Srwatson 262239674Srwatson if (sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL) 263239674Srwatson continue; 264239674Srwatson 265239674Srwatson err = pvr_check_sync_op(&sync_ops[i]); 266239674Srwatson if (err) 267239674Srwatson return err; 268239674Srwatson 269239674Srwatson sig_sync = pvr_sync_signal_array_search(signal_array, sync_ops[i].handle, 270239674Srwatson sync_ops[i].value); 271239674Srwatson if (sig_sync) { 272239674Srwatson if (WARN_ON(!sig_sync->fence)) 273239674Srwatson return -EINVAL; 274239674Srwatson 275239674Srwatson fence = dma_fence_get(sig_sync->fence); 276239674Srwatson } else { 277239674Srwatson err = drm_syncobj_find_fence(from_pvr_file(pvr_file), sync_ops[i].handle, 278239674Srwatson sync_ops[i].value, 0, &fence); 279239674Srwatson if (err) 280239674Srwatson return err; 281239674Srwatson } 282239674Srwatson 283239674Srwatson err = pvr_sync_add_dep_to_job(job, fence); 284239674Srwatson if (err) 285239674Srwatson return err; 286239674Srwatson } 287239674Srwatson 288239674Srwatson return 0; 289239674Srwatson} 290239674Srwatson