13229Spst// SPDX-License-Identifier: GPL-2.0-only OR MIT 23229Spst/* Copyright (c) 2023 Imagination Technologies Ltd. */ 33229Spst 43229Spst#include "pvr_cccb.h" 53229Spst#include "pvr_context.h" 63229Spst#include "pvr_device.h" 73229Spst#include "pvr_drv.h" 83229Spst#include "pvr_gem.h" 93229Spst#include "pvr_job.h" 103229Spst#include "pvr_power.h" 113229Spst#include "pvr_rogue_fwif.h" 123229Spst#include "pvr_rogue_fwif_common.h" 133229Spst#include "pvr_rogue_fwif_resetframework.h" 143229Spst#include "pvr_stream.h" 153229Spst#include "pvr_stream_defs.h" 163229Spst#include "pvr_vm.h" 173229Spst 183229Spst#include <drm/drm_auth.h> 193229Spst#include <drm/drm_managed.h> 203229Spst#include <linux/errno.h> 2118471Swosch#include <linux/kernel.h> 223229Spst#include <linux/sched.h> 233229Spst#include <linux/slab.h> 243229Spst#include <linux/string.h> 253229Spst#include <linux/types.h> 263229Spst#include <linux/xarray.h> 273229Spst 283229Spststatic int 293229Spstremap_priority(struct pvr_file *pvr_file, s32 uapi_priority, 303229Spst enum pvr_context_priority *priority_out) 313229Spst{ 323229Spst switch (uapi_priority) { 333229Spst case DRM_PVR_CTX_PRIORITY_LOW: 343229Spst *priority_out = PVR_CTX_PRIORITY_LOW; 353229Spst break; 363229Spst case DRM_PVR_CTX_PRIORITY_NORMAL: 373229Spst *priority_out = PVR_CTX_PRIORITY_MEDIUM; 383229Spst break; 393229Spst case DRM_PVR_CTX_PRIORITY_HIGH: 40110395Scharnier if (!capable(CAP_SYS_NICE) && !drm_is_current_master(from_pvr_file(pvr_file))) 41110395Scharnier return -EACCES; 423229Spst *priority_out = PVR_CTX_PRIORITY_HIGH; 433229Spst break; 443229Spst default: 453229Spst return -EINVAL; 463229Spst } 473229Spst 483229Spst return 0; 493229Spst} 5013575Spst 513229Spststatic int get_fw_obj_size(enum drm_pvr_ctx_type type) 523229Spst{ 533229Spst switch (type) { 543229Spst case DRM_PVR_CTX_TYPE_RENDER: 553229Spst return sizeof(struct rogue_fwif_fwrendercontext); 563229Spst case DRM_PVR_CTX_TYPE_COMPUTE: 573229Spst return sizeof(struct rogue_fwif_fwcomputecontext); 583229Spst case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 5913575Spst return sizeof(struct rogue_fwif_fwtransfercontext); 603229Spst } 613229Spst 623229Spst return -EINVAL; 633229Spst} 643229Spst 653229Spststatic int 663229Spstprocess_static_context_state(struct pvr_device *pvr_dev, const struct pvr_stream_cmd_defs *cmd_defs, 6769793Sobrien u64 stream_user_ptr, u32 stream_size, void *dest) 683229Spst{ 693229Spst void *stream; 70153706Strhodes int err; 713229Spst 723229Spst stream = kzalloc(stream_size, GFP_KERNEL); 733229Spst if (!stream) 743229Spst return -ENOMEM; 753229Spst 763229Spst if (copy_from_user(stream, u64_to_user_ptr(stream_user_ptr), stream_size)) { 773229Spst err = -EFAULT; 783229Spst goto err_free; 793229Spst } 803229Spst 813229Spst err = pvr_stream_process(pvr_dev, cmd_defs, stream, stream_size, dest); 823229Spst if (err) 833229Spst goto err_free; 843229Spst 853229Spst kfree(stream); 863229Spst 873229Spst return 0; 883229Spst 893229Spsterr_free: 903229Spst kfree(stream); 913229Spst 923229Spst return err; 933229Spst} 943229Spst 953229Spststatic int init_render_fw_objs(struct pvr_context *ctx, 963229Spst struct drm_pvr_ioctl_create_context_args *args, 973229Spst void *fw_ctx_map) 983229Spst{ 993229Spst struct rogue_fwif_static_rendercontext_state *static_rendercontext_state; 1003229Spst struct rogue_fwif_fwrendercontext *fw_render_context = fw_ctx_map; 1013229Spst 1023229Spst if (!args->static_context_state_len) 1033229Spst return -EINVAL; 1043229Spst 1053229Spst static_rendercontext_state = &fw_render_context->static_render_context_state; 1063229Spst 1073229Spst /* Copy static render context state from userspace. */ 10897417Salfred return process_static_context_state(ctx->pvr_dev, 1093229Spst &pvr_static_render_context_state_stream, 11097417Salfred args->static_context_state, 11197417Salfred args->static_context_state_len, 1123229Spst &static_rendercontext_state->ctxswitch_regs[0]); 11397417Salfred} 1143229Spst 11597417Salfredstatic int init_compute_fw_objs(struct pvr_context *ctx, 11697417Salfred struct drm_pvr_ioctl_create_context_args *args, 11797417Salfred void *fw_ctx_map) 11897417Salfred{ 11997417Salfred struct rogue_fwif_fwcomputecontext *fw_compute_context = fw_ctx_map; 1203229Spst struct rogue_fwif_cdm_registers_cswitch *ctxswitch_regs; 1213229Spst 1223229Spst if (!args->static_context_state_len) 1233229Spst return -EINVAL; 1243229Spst 1253229Spst ctxswitch_regs = &fw_compute_context->static_compute_context_state.ctxswitch_regs; 1263229Spst 1273229Spst /* Copy static render context state from userspace. */ 1283229Spst return process_static_context_state(ctx->pvr_dev, 1293229Spst &pvr_static_compute_context_state_stream, 1303229Spst args->static_context_state, 1313229Spst args->static_context_state_len, 1323229Spst ctxswitch_regs); 1333229Spst} 1343229Spst 1353229Spststatic int init_transfer_fw_objs(struct pvr_context *ctx, 1363229Spst struct drm_pvr_ioctl_create_context_args *args, 1373229Spst void *fw_ctx_map) 1383229Spst{ 1393229Spst if (args->static_context_state_len) 1403229Spst return -EINVAL; 1413229Spst 1423229Spst return 0; 1433229Spst} 1443229Spst 1453229Spststatic int init_fw_objs(struct pvr_context *ctx, 1463229Spst struct drm_pvr_ioctl_create_context_args *args, 1473229Spst void *fw_ctx_map) 1483229Spst{ 1493229Spst switch (ctx->type) { 1503229Spst case DRM_PVR_CTX_TYPE_RENDER: 1513229Spst return init_render_fw_objs(ctx, args, fw_ctx_map); 1523229Spst case DRM_PVR_CTX_TYPE_COMPUTE: 1533229Spst return init_compute_fw_objs(ctx, args, fw_ctx_map); 1543229Spst case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 1553229Spst return init_transfer_fw_objs(ctx, args, fw_ctx_map); 1563229Spst } 1573229Spst 15845422Sbrian return -EINVAL; 15945422Sbrian} 16013575Spst 1613229Spststatic void 1623229Spstctx_fw_data_init(void *cpu_ptr, void *priv) 1633229Spst{ 1643229Spst struct pvr_context *ctx = priv; 1653229Spst 1663229Spst memcpy(cpu_ptr, ctx->data, ctx->data_size); 1673229Spst} 1683229Spst 1693229Spst/** 1703229Spst * pvr_context_destroy_queues() - Destroy all queues attached to a context. 1713229Spst * @ctx: Context to destroy queues on. 1723229Spst * 1733229Spst * Should be called when the last reference to a context object is dropped. 1743229Spst * It releases all resources attached to the queues bound to this context. 1753229Spst */ 1763229Spststatic void pvr_context_destroy_queues(struct pvr_context *ctx) 1773229Spst{ 1783229Spst switch (ctx->type) { 17946078Simp case DRM_PVR_CTX_TYPE_RENDER: 1803229Spst pvr_queue_destroy(ctx->queues.fragment); 1813229Spst pvr_queue_destroy(ctx->queues.geometry); 1823229Spst break; 1833229Spst case DRM_PVR_CTX_TYPE_COMPUTE: 1843229Spst pvr_queue_destroy(ctx->queues.compute); 1853229Spst break; 1863229Spst case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 1873229Spst pvr_queue_destroy(ctx->queues.transfer); 1883229Spst break; 189141918Sstefanf } 190141918Sstefanf} 191116370Sjmg 192116370Sjmg/** 1933229Spst * pvr_context_create_queues() - Create all queues attached to a context. 19413575Spst * @ctx: Context to create queues on. 19513575Spst * @args: Context creation arguments passed by userspace. 19613575Spst * @fw_ctx_map: CPU mapping of the FW context object. 1973229Spst * 1983229Spst * Return: 1993229Spst * * 0 on success, or 2003229Spst * * A negative error code otherwise. 2013229Spst */ 2023229Spststatic int pvr_context_create_queues(struct pvr_context *ctx, 2033229Spst struct drm_pvr_ioctl_create_context_args *args, 2043229Spst void *fw_ctx_map) 2053229Spst{ 2063229Spst int err; 2073229Spst 2083229Spst switch (ctx->type) { 2093229Spst case DRM_PVR_CTX_TYPE_RENDER: 2103229Spst ctx->queues.geometry = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_GEOMETRY, 2113229Spst args, fw_ctx_map); 2123229Spst if (IS_ERR(ctx->queues.geometry)) { 2133229Spst err = PTR_ERR(ctx->queues.geometry); 2143229Spst ctx->queues.geometry = NULL; 2153229Spst goto err_destroy_queues; 2163229Spst } 2173229Spst 2183229Spst ctx->queues.fragment = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_FRAGMENT, 2193229Spst args, fw_ctx_map); 2203229Spst if (IS_ERR(ctx->queues.fragment)) { 2213229Spst err = PTR_ERR(ctx->queues.fragment); 2223229Spst ctx->queues.fragment = NULL; 2233229Spst goto err_destroy_queues; 2243229Spst } 2253229Spst return 0; 2263229Spst 2273229Spst case DRM_PVR_CTX_TYPE_COMPUTE: 2283229Spst ctx->queues.compute = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_COMPUTE, 2293229Spst args, fw_ctx_map); 2303229Spst if (IS_ERR(ctx->queues.compute)) { 2313229Spst err = PTR_ERR(ctx->queues.compute); 2323229Spst ctx->queues.compute = NULL; 2333229Spst goto err_destroy_queues; 2343229Spst } 2353229Spst return 0; 2363229Spst 2373229Spst case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 2383229Spst ctx->queues.transfer = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_TRANSFER_FRAG, 2393229Spst args, fw_ctx_map); 2403229Spst if (IS_ERR(ctx->queues.transfer)) { 2413229Spst err = PTR_ERR(ctx->queues.transfer); 2423229Spst ctx->queues.transfer = NULL; 2433229Spst goto err_destroy_queues; 2443229Spst } 2453229Spst return 0; 2463229Spst } 2473229Spst 2483229Spst return -EINVAL; 2493229Spst 2503229Spsterr_destroy_queues: 2513229Spst pvr_context_destroy_queues(ctx); 2523229Spst return err; 2533229Spst} 25445422Sbrian 25525717Sphk/** 25613575Spst * pvr_context_kill_queues() - Kill queues attached to context. 25713575Spst * @ctx: Context to kill queues on. 25845422Sbrian * 25936617Sjoerg * Killing the queues implies making them unusable for future jobs, while still 26013575Spst * letting the currently submitted jobs a chance to finish. Queue resources will 2613229Spst * stay around until pvr_context_destroy_queues() is called. 2623229Spst */ 2633229Spststatic void pvr_context_kill_queues(struct pvr_context *ctx) 2643229Spst{ 2653229Spst switch (ctx->type) { 2663229Spst case DRM_PVR_CTX_TYPE_RENDER: 2673229Spst pvr_queue_kill(ctx->queues.fragment); 2683229Spst pvr_queue_kill(ctx->queues.geometry); 2693229Spst break; 2703229Spst case DRM_PVR_CTX_TYPE_COMPUTE: 2713229Spst pvr_queue_kill(ctx->queues.compute); 2723229Spst break; 2733229Spst case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: 2743229Spst pvr_queue_kill(ctx->queues.transfer); 2753229Spst break; 2763229Spst } 2773229Spst} 27825717Sphk 2793229Spst/** 2803229Spst * pvr_context_create() - Create a context. 2813229Spst * @pvr_file: File to attach the created context to. 2823229Spst * @args: Context creation arguments. 2833229Spst * 2843229Spst * Return: 2853229Spst * * 0 on success, or 2863229Spst * * A negative error code on failure. 2873229Spst */ 2883229Spstint pvr_context_create(struct pvr_file *pvr_file, struct drm_pvr_ioctl_create_context_args *args) 2893229Spst{ 2903229Spst struct pvr_device *pvr_dev = pvr_file->pvr_dev; 2913229Spst struct pvr_context *ctx; 2923229Spst int ctx_size; 2933229Spst int err; 2943229Spst 2953229Spst /* Context creation flags are currently unused and must be zero. */ 2963229Spst if (args->flags) 2973229Spst return -EINVAL; 2983229Spst 2993229Spst ctx_size = get_fw_obj_size(args->type); 3003229Spst if (ctx_size < 0) 30125717Sphk return ctx_size; 3023229Spst 3033229Spst ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 3043229Spst if (!ctx) 3053229Spst return -ENOMEM; 3063229Spst 3073229Spst ctx->data_size = ctx_size; 3083229Spst ctx->type = args->type; 3093229Spst ctx->flags = args->flags; 3103229Spst ctx->pvr_dev = pvr_dev; 3113229Spst kref_init(&ctx->ref_count); 3123229Spst 3133229Spst err = remap_priority(pvr_file, args->priority, &ctx->priority); 3143229Spst if (err) 3153229Spst goto err_free_ctx; 3163229Spst 31725717Sphk ctx->vm_ctx = pvr_vm_context_lookup(pvr_file, args->vm_context_handle); 3183229Spst if (IS_ERR(ctx->vm_ctx)) { 3193229Spst err = PTR_ERR(ctx->vm_ctx); 3203229Spst goto err_free_ctx; 32113575Spst } 3223229Spst 3233229Spst ctx->data = kzalloc(ctx_size, GFP_KERNEL); 3243229Spst if (!ctx->data) { 3253229Spst err = -ENOMEM; 3263229Spst goto err_put_vm; 3273229Spst } 3283229Spst 3293229Spst err = pvr_context_create_queues(ctx, args, ctx->data); 3303229Spst if (err) 3313229Spst goto err_free_ctx_data; 3323229Spst 3333229Spst err = init_fw_objs(ctx, args, ctx->data); 3343229Spst if (err) 3353229Spst goto err_destroy_queues; 3363229Spst 3373229Spst err = pvr_fw_object_create(pvr_dev, ctx_size, PVR_BO_FW_FLAGS_DEVICE_UNCACHED, 3383229Spst ctx_fw_data_init, ctx, &ctx->fw_obj); 3393229Spst if (err) 3403229Spst goto err_free_ctx_data; 34125717Sphk 3423229Spst err = xa_alloc(&pvr_dev->ctx_ids, &ctx->ctx_id, ctx, xa_limit_32b, GFP_KERNEL); 3433229Spst if (err) 3443229Spst goto err_destroy_fw_obj; 3453229Spst 3463229Spst err = xa_alloc(&pvr_file->ctx_handles, &args->handle, ctx, xa_limit_32b, GFP_KERNEL); 3473229Spst if (err) { 3483229Spst /* 3493229Spst * It's possible that another thread could have taken a reference on the context at 3503229Spst * this point as it is in the ctx_ids xarray. Therefore instead of directly 3513229Spst * destroying the context, drop a reference instead. 3523229Spst */ 3533229Spst pvr_context_put(ctx); 3543229Spst return err; 35525717Sphk } 3563229Spst 3573229Spst return 0; 3583229Spst 3593229Spsterr_destroy_fw_obj: 3603229Spst pvr_fw_object_destroy(ctx->fw_obj); 3613229Spst 3623229Spsterr_destroy_queues: 3633229Spst pvr_context_destroy_queues(ctx); 3643229Spst 3653229Spsterr_free_ctx_data: 3663229Spst kfree(ctx->data); 3673229Spst 3683229Spsterr_put_vm: 3693229Spst pvr_vm_context_put(ctx->vm_ctx); 3703229Spst 3713229Spsterr_free_ctx: 3723229Spst kfree(ctx); 3733229Spst return err; 3743229Spst} 37513575Spst 3763229Spststatic void 3773229Spstpvr_context_release(struct kref *ref_count) 37825717Sphk{ 3793229Spst struct pvr_context *ctx = 3803229Spst container_of(ref_count, struct pvr_context, ref_count); 3813229Spst struct pvr_device *pvr_dev = ctx->pvr_dev; 3823229Spst 3833229Spst xa_erase(&pvr_dev->ctx_ids, ctx->ctx_id); 3843229Spst pvr_context_destroy_queues(ctx); 3853229Spst pvr_fw_object_destroy(ctx->fw_obj); 3863229Spst kfree(ctx->data); 3873229Spst pvr_vm_context_put(ctx->vm_ctx); 3883229Spst kfree(ctx); 3893229Spst} 3903229Spst 3913229Spst/** 3923229Spst * pvr_context_put() - Release reference on context 39369793Sobrien * @ctx: Target context. 3943229Spst */ 3953229Spstvoid 3963229Spstpvr_context_put(struct pvr_context *ctx) 3973229Spst{ 3983229Spst if (ctx) 3993229Spst kref_put(&ctx->ref_count, pvr_context_release); 4003229Spst} 4013229Spst 4023229Spst/** 4033229Spst * pvr_context_destroy() - Destroy context 4043229Spst * @pvr_file: Pointer to pvr_file structure. 4053229Spst * @handle: Userspace context handle. 4063229Spst * 4073229Spst * Removes context from context list and drops initial reference. Context will 4083229Spst * then be destroyed once all outstanding references are dropped. 4093229Spst * 4103229Spst * Return: 4113229Spst * * 0 on success, or 4123229Spst * * -%EINVAL if context not in context list. 4133229Spst */ 4143229Spstint 4153229Spstpvr_context_destroy(struct pvr_file *pvr_file, u32 handle) 4163229Spst{ 4173229Spst struct pvr_context *ctx = xa_erase(&pvr_file->ctx_handles, handle); 4183229Spst 4193229Spst if (!ctx) 4203229Spst return -EINVAL; 4213229Spst 4223229Spst /* Make sure nothing can be queued to the queues after that point. */ 4233229Spst pvr_context_kill_queues(ctx); 4243229Spst 4253229Spst /* Release the reference held by the handle set. */ 4263229Spst pvr_context_put(ctx); 4273229Spst 4283229Spst return 0; 4293229Spst} 4303229Spst 4313229Spst/** 4323229Spst * pvr_destroy_contexts_for_file: Destroy any contexts associated with the given file 4333229Spst * @pvr_file: Pointer to pvr_file structure. 4343229Spst * 4353229Spst * Removes all contexts associated with @pvr_file from the device context list and drops initial 4363229Spst * references. Contexts will then be destroyed once all outstanding references are dropped. 4373229Spst */ 4383229Spstvoid pvr_destroy_contexts_for_file(struct pvr_file *pvr_file) 4393229Spst{ 4403229Spst struct pvr_context *ctx; 4413229Spst unsigned long handle; 4423229Spst 4433229Spst xa_for_each(&pvr_file->ctx_handles, handle, ctx) 4443229Spst pvr_context_destroy(pvr_file, handle); 4453229Spst} 4463229Spst 4473229Spst/** 448110395Scharnier * pvr_context_device_init() - Device level initialization for queue related resources. 4493229Spst * @pvr_dev: The device to initialize. 4503229Spst */ 4513229Spstvoid pvr_context_device_init(struct pvr_device *pvr_dev) 4523229Spst{ 4533229Spst xa_init_flags(&pvr_dev->ctx_ids, XA_FLAGS_ALLOC1); 4543229Spst} 4553229Spst 4563229Spst/** 4573229Spst * pvr_context_device_fini() - Device level cleanup for queue related resources. 4583229Spst * @pvr_dev: The device to cleanup. 4593229Spst */ 4603229Spstvoid pvr_context_device_fini(struct pvr_device *pvr_dev) 4613229Spst{ 4623229Spst WARN_ON(!xa_empty(&pvr_dev->ctx_ids)); 4633229Spst xa_destroy(&pvr_dev->ctx_ids); 4643229Spst} 4653229Spst