diff options
author | Jonathan Gray <jsg@cvs.openbsd.org> | 2022-09-02 05:47:02 +0000 |
---|---|---|
committer | Jonathan Gray <jsg@cvs.openbsd.org> | 2022-09-02 05:47:02 +0000 |
commit | 0dbbf1e0708df85a357d70e2708c0a11aeb5480e (patch) | |
tree | 6656ff8eb8b15a2fc1c02888973caf618388cfd0 /lib/mesa/src/freedreno/drm | |
parent | 5f66494d31f735486b8222ecfa0a0c9046e92543 (diff) |
Merge Mesa 22.1.7
Diffstat (limited to 'lib/mesa/src/freedreno/drm')
-rw-r--r-- | lib/mesa/src/freedreno/drm/msm_bo.c | 197 | ||||
-rw-r--r-- | lib/mesa/src/freedreno/drm/msm_device.c | 61 | ||||
-rw-r--r-- | lib/mesa/src/freedreno/drm/msm_pipe.c | 220 | ||||
-rw-r--r-- | lib/mesa/src/freedreno/drm/msm_priv.h | 140 | ||||
-rw-r--r-- | lib/mesa/src/freedreno/drm/msm_ringbuffer.c | 722 | ||||
-rw-r--r-- | lib/mesa/src/freedreno/drm/msm_ringbuffer_sp.c | 568 | ||||
-rw-r--r-- | lib/mesa/src/freedreno/drm/msm_ringbuffer_sp.h | 140 |
7 files changed, 0 insertions, 2048 deletions
diff --git a/lib/mesa/src/freedreno/drm/msm_bo.c b/lib/mesa/src/freedreno/drm/msm_bo.c deleted file mode 100644 index da2609903..000000000 --- a/lib/mesa/src/freedreno/drm/msm_bo.c +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright (C) 2012-2018 Rob Clark <robclark@freedesktop.org> - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * Authors: - * Rob Clark <robclark@freedesktop.org> - */ - -#include "msm_priv.h" - -static int bo_allocate(struct msm_bo *msm_bo) -{ - struct fd_bo *bo = &msm_bo->base; - if (!msm_bo->offset) { - struct drm_msm_gem_info req = { - .handle = bo->handle, - .info = MSM_INFO_GET_OFFSET, - }; - int ret; - - /* if the buffer is already backed by pages then this - * doesn't actually do anything (other than giving us - * the offset) - */ - ret = drmCommandWriteRead(bo->dev->fd, DRM_MSM_GEM_INFO, - &req, sizeof(req)); - if (ret) { - ERROR_MSG("alloc failed: %s", strerror(errno)); - return ret; - } - - msm_bo->offset = req.value; - } - - return 0; -} - -static int msm_bo_offset(struct fd_bo *bo, uint64_t *offset) -{ - struct msm_bo *msm_bo = to_msm_bo(bo); - int ret = bo_allocate(msm_bo); - if (ret) - return ret; - *offset = msm_bo->offset; - return 0; -} - -static int msm_bo_cpu_prep(struct fd_bo *bo, struct fd_pipe *pipe, uint32_t op) -{ - struct drm_msm_gem_cpu_prep req = { - .handle = bo->handle, - .op = op, - }; - - get_abs_timeout(&req.timeout, 5000000000); - - return drmCommandWrite(bo->dev->fd, DRM_MSM_GEM_CPU_PREP, &req, sizeof(req)); -} - -static void msm_bo_cpu_fini(struct fd_bo *bo) -{ - struct drm_msm_gem_cpu_fini req = { - .handle = bo->handle, - }; - - drmCommandWrite(bo->dev->fd, DRM_MSM_GEM_CPU_FINI, &req, sizeof(req)); -} - -static int msm_bo_madvise(struct fd_bo *bo, int willneed) -{ - struct drm_msm_gem_madvise req = { - .handle = bo->handle, - .madv = willneed ? MSM_MADV_WILLNEED : MSM_MADV_DONTNEED, - }; - int ret; - - /* older kernels do not support this: */ - if (bo->dev->version < FD_VERSION_MADVISE) - return willneed; - - ret = drmCommandWriteRead(bo->dev->fd, DRM_MSM_GEM_MADVISE, &req, sizeof(req)); - if (ret) - return ret; - - return req.retained; -} - -static uint64_t msm_bo_iova(struct fd_bo *bo) -{ - struct drm_msm_gem_info req = { - .handle = bo->handle, - .info = MSM_INFO_GET_IOVA, - }; - int ret; - - ret = drmCommandWriteRead(bo->dev->fd, DRM_MSM_GEM_INFO, &req, sizeof(req)); - debug_assert(ret == 0); - - return req.value; -} - -static void msm_bo_set_name(struct fd_bo *bo, const char *fmt, va_list ap) -{ - struct drm_msm_gem_info req = { - .handle = bo->handle, - .info = MSM_INFO_SET_NAME, - }; - char buf[32]; - int sz; - - if (bo->dev->version < FD_VERSION_SOFTPIN) - return; - - sz = vsnprintf(buf, sizeof(buf), fmt, ap); - - req.value = VOID2U64(buf); - req.len = MIN2(sz, sizeof(buf)); - - drmCommandWrite(bo->dev->fd, DRM_MSM_GEM_INFO, &req, sizeof(req)); -} - -static void msm_bo_destroy(struct fd_bo *bo) -{ - struct msm_bo *msm_bo = to_msm_bo(bo); - free(msm_bo); -} - -static const struct fd_bo_funcs funcs = { - .offset = msm_bo_offset, - .cpu_prep = msm_bo_cpu_prep, - .cpu_fini = msm_bo_cpu_fini, - .madvise = msm_bo_madvise, - .iova = msm_bo_iova, - .set_name = msm_bo_set_name, - .destroy = msm_bo_destroy, -}; - -/* allocate a buffer handle: */ -int msm_bo_new_handle(struct fd_device *dev, - uint32_t size, uint32_t flags, uint32_t *handle) -{ - struct drm_msm_gem_new req = { - .size = size, - .flags = MSM_BO_WC, // TODO figure out proper flags.. - }; - int ret; - - if (flags & DRM_FREEDRENO_GEM_SCANOUT) - req.flags |= MSM_BO_SCANOUT; - - if (flags & DRM_FREEDRENO_GEM_GPUREADONLY) - req.flags |= MSM_BO_GPU_READONLY; - - ret = drmCommandWriteRead(dev->fd, DRM_MSM_GEM_NEW, - &req, sizeof(req)); - if (ret) - return ret; - - *handle = req.handle; - - return 0; -} - -/* allocate a new buffer object */ -struct fd_bo * msm_bo_from_handle(struct fd_device *dev, - uint32_t size, uint32_t handle) -{ - struct msm_bo *msm_bo; - struct fd_bo *bo; - - msm_bo = calloc(1, sizeof(*msm_bo)); - if (!msm_bo) - return NULL; - - bo = &msm_bo->base; - bo->funcs = &funcs; - - return bo; -} diff --git a/lib/mesa/src/freedreno/drm/msm_device.c b/lib/mesa/src/freedreno/drm/msm_device.c deleted file mode 100644 index d391ef013..000000000 --- a/lib/mesa/src/freedreno/drm/msm_device.c +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2012-2018 Rob Clark <robclark@freedesktop.org> - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * Authors: - * Rob Clark <robclark@freedesktop.org> - */ - -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> - -#include "msm_priv.h" - -static void msm_device_destroy(struct fd_device *dev) -{ - struct msm_device *msm_dev = to_msm_device(dev); - free(msm_dev); -} - -static const struct fd_device_funcs funcs = { - .bo_new_handle = msm_bo_new_handle, - .bo_from_handle = msm_bo_from_handle, - .pipe_new = msm_pipe_new, - .destroy = msm_device_destroy, -}; - -struct fd_device * msm_device_new(int fd) -{ - struct msm_device *msm_dev; - struct fd_device *dev; - - msm_dev = calloc(1, sizeof(*msm_dev)); - if (!msm_dev) - return NULL; - - dev = &msm_dev->base; - dev->funcs = &funcs; - - dev->bo_size = sizeof(struct msm_bo); - - return dev; -} diff --git a/lib/mesa/src/freedreno/drm/msm_pipe.c b/lib/mesa/src/freedreno/drm/msm_pipe.c deleted file mode 100644 index 7d5b9fcd7..000000000 --- a/lib/mesa/src/freedreno/drm/msm_pipe.c +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright (C) 2012-2018 Rob Clark <robclark@freedesktop.org> - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * Authors: - * Rob Clark <robclark@freedesktop.org> - */ - -#include "util/slab.h" - -#include "msm_priv.h" - -static int query_param(struct fd_pipe *pipe, uint32_t param, - uint64_t *value) -{ - struct msm_pipe *msm_pipe = to_msm_pipe(pipe); - struct drm_msm_param req = { - .pipe = msm_pipe->pipe, - .param = param, - }; - int ret; - - ret = drmCommandWriteRead(pipe->dev->fd, DRM_MSM_GET_PARAM, - &req, sizeof(req)); - if (ret) - return ret; - - *value = req.value; - - return 0; -} - -static int msm_pipe_get_param(struct fd_pipe *pipe, - enum fd_param_id param, uint64_t *value) -{ - struct msm_pipe *msm_pipe = to_msm_pipe(pipe); - switch(param) { - case FD_DEVICE_ID: // XXX probably get rid of this.. - case FD_GPU_ID: - *value = msm_pipe->gpu_id; - return 0; - case FD_GMEM_SIZE: - *value = msm_pipe->gmem; - return 0; - case FD_CHIP_ID: - *value = msm_pipe->chip_id; - return 0; - case FD_MAX_FREQ: - return query_param(pipe, MSM_PARAM_MAX_FREQ, value); - case FD_TIMESTAMP: - return query_param(pipe, MSM_PARAM_TIMESTAMP, value); - case FD_NR_RINGS: - return query_param(pipe, MSM_PARAM_NR_RINGS, value); - default: - ERROR_MSG("invalid param id: %d", param); - return -1; - } -} - -static int msm_pipe_wait(struct fd_pipe *pipe, uint32_t timestamp, - uint64_t timeout) -{ - struct fd_device *dev = pipe->dev; - struct drm_msm_wait_fence req = { - .fence = timestamp, - .queueid = to_msm_pipe(pipe)->queue_id, - }; - int ret; - - get_abs_timeout(&req.timeout, timeout); - - ret = drmCommandWrite(dev->fd, DRM_MSM_WAIT_FENCE, &req, sizeof(req)); - if (ret) { - ERROR_MSG("wait-fence failed! %d (%s)", ret, strerror(errno)); - return ret; - } - - return 0; -} - -static int open_submitqueue(struct fd_pipe *pipe, uint32_t prio) -{ - struct drm_msm_submitqueue req = { - .flags = 0, - .prio = prio, - }; - uint64_t nr_rings = 1; - int ret; - - if (fd_device_version(pipe->dev) < FD_VERSION_SUBMIT_QUEUES) { - to_msm_pipe(pipe)->queue_id = 0; - return 0; - } - - msm_pipe_get_param(pipe, FD_NR_RINGS, &nr_rings); - - req.prio = MIN2(req.prio, MAX2(nr_rings, 1) - 1); - - ret = drmCommandWriteRead(pipe->dev->fd, DRM_MSM_SUBMITQUEUE_NEW, - &req, sizeof(req)); - if (ret) { - ERROR_MSG("could not create submitqueue! %d (%s)", ret, strerror(errno)); - return ret; - } - - to_msm_pipe(pipe)->queue_id = req.id; - return 0; -} - -static void close_submitqueue(struct fd_pipe *pipe, uint32_t queue_id) -{ - if (fd_device_version(pipe->dev) < FD_VERSION_SUBMIT_QUEUES) - return; - - drmCommandWrite(pipe->dev->fd, DRM_MSM_SUBMITQUEUE_CLOSE, - &queue_id, sizeof(queue_id)); -} - -static void msm_pipe_destroy(struct fd_pipe *pipe) -{ - struct msm_pipe *msm_pipe = to_msm_pipe(pipe); - close_submitqueue(pipe, msm_pipe->queue_id); - free(msm_pipe); -} - -static const struct fd_pipe_funcs sp_funcs = { - .ringbuffer_new_object = msm_ringbuffer_sp_new_object, - .submit_new = msm_submit_sp_new, - .get_param = msm_pipe_get_param, - .wait = msm_pipe_wait, - .destroy = msm_pipe_destroy, -}; - -static const struct fd_pipe_funcs legacy_funcs = { - .ringbuffer_new_object = msm_ringbuffer_new_object, - .submit_new = msm_submit_new, - .get_param = msm_pipe_get_param, - .wait = msm_pipe_wait, - .destroy = msm_pipe_destroy, -}; - -static uint64_t get_param(struct fd_pipe *pipe, uint32_t param) -{ - uint64_t value; - int ret = query_param(pipe, param, &value); - if (ret) { - ERROR_MSG("get-param failed! %d (%s)", ret, strerror(errno)); - return 0; - } - return value; -} - -struct fd_pipe * msm_pipe_new(struct fd_device *dev, - enum fd_pipe_id id, uint32_t prio) -{ - static const uint32_t pipe_id[] = { - [FD_PIPE_3D] = MSM_PIPE_3D0, - [FD_PIPE_2D] = MSM_PIPE_2D0, - }; - struct msm_pipe *msm_pipe = NULL; - struct fd_pipe *pipe = NULL; - - msm_pipe = calloc(1, sizeof(*msm_pipe)); - if (!msm_pipe) { - ERROR_MSG("allocation failed"); - goto fail; - } - - pipe = &msm_pipe->base; - - if (fd_device_version(dev) >= FD_VERSION_SOFTPIN) { - pipe->funcs = &sp_funcs; - } else { - pipe->funcs = &legacy_funcs; - } - - /* initialize before get_param(): */ - pipe->dev = dev; - msm_pipe->pipe = pipe_id[id]; - - /* these params should be supported since the first version of drm/msm: */ - msm_pipe->gpu_id = get_param(pipe, MSM_PARAM_GPU_ID); - msm_pipe->gmem = get_param(pipe, MSM_PARAM_GMEM_SIZE); - msm_pipe->chip_id = get_param(pipe, MSM_PARAM_CHIP_ID); - - if (! msm_pipe->gpu_id) - goto fail; - - INFO_MSG("Pipe Info:"); - INFO_MSG(" GPU-id: %d", msm_pipe->gpu_id); - INFO_MSG(" Chip-id: 0x%08x", msm_pipe->chip_id); - INFO_MSG(" GMEM size: 0x%08x", msm_pipe->gmem); - - if (open_submitqueue(pipe, prio)) - goto fail; - - return pipe; -fail: - if (pipe) - fd_pipe_del(pipe); - return NULL; -} diff --git a/lib/mesa/src/freedreno/drm/msm_priv.h b/lib/mesa/src/freedreno/drm/msm_priv.h deleted file mode 100644 index 9cb60bc1d..000000000 --- a/lib/mesa/src/freedreno/drm/msm_priv.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (C) 2012-2018 Rob Clark <robclark@freedesktop.org> - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * Authors: - * Rob Clark <robclark@freedesktop.org> - */ - -#ifndef MSM_PRIV_H_ -#define MSM_PRIV_H_ - -#include "freedreno_priv.h" - -#ifndef __user -# define __user -#endif - -#include "msm_drm.h" - -struct msm_device { - struct fd_device base; - struct fd_bo_cache ring_cache; -}; -FD_DEFINE_CAST(fd_device, msm_device); - -struct fd_device * msm_device_new(int fd); - -struct msm_pipe { - struct fd_pipe base; - uint32_t pipe; - uint32_t gpu_id; - uint32_t gmem; - uint32_t chip_id; - uint32_t queue_id; -}; -FD_DEFINE_CAST(fd_pipe, msm_pipe); - -struct fd_pipe * msm_pipe_new(struct fd_device *dev, - enum fd_pipe_id id, uint32_t prio); - -struct fd_ringbuffer * msm_ringbuffer_new_object(struct fd_pipe *pipe, uint32_t size); -struct fd_ringbuffer * msm_ringbuffer_sp_new_object(struct fd_pipe *pipe, uint32_t size); - -struct fd_submit * msm_submit_new(struct fd_pipe *pipe); -struct fd_submit * msm_submit_sp_new(struct fd_pipe *pipe); - -struct msm_bo { - struct fd_bo base; - uint64_t offset; - /* to avoid excess hashtable lookups, cache the ring this bo was - * last emitted on (since that will probably also be the next ring - * it is emitted on) - */ - unsigned current_submit_seqno; - uint32_t idx; -}; -FD_DEFINE_CAST(fd_bo, msm_bo); - -int msm_bo_new_handle(struct fd_device *dev, - uint32_t size, uint32_t flags, uint32_t *handle); -struct fd_bo * msm_bo_from_handle(struct fd_device *dev, - uint32_t size, uint32_t handle); - -static inline void -msm_dump_submit(struct drm_msm_gem_submit *req) -{ - for (unsigned i = 0; i < req->nr_bos; i++) { - struct drm_msm_gem_submit_bo *bos = U642VOID(req->bos); - struct drm_msm_gem_submit_bo *bo = &bos[i]; - ERROR_MSG(" bos[%d]: handle=%u, flags=%x", i, bo->handle, bo->flags); - } - for (unsigned i = 0; i < req->nr_cmds; i++) { - struct drm_msm_gem_submit_cmd *cmds = U642VOID(req->cmds); - struct drm_msm_gem_submit_cmd *cmd = &cmds[i]; - struct drm_msm_gem_submit_reloc *relocs = U642VOID(cmd->relocs); - ERROR_MSG(" cmd[%d]: type=%u, submit_idx=%u, submit_offset=%u, size=%u", - i, cmd->type, cmd->submit_idx, cmd->submit_offset, cmd->size); - for (unsigned j = 0; j < cmd->nr_relocs; j++) { - struct drm_msm_gem_submit_reloc *r = &relocs[j]; - ERROR_MSG(" reloc[%d]: submit_offset=%u, or=%08x, shift=%d, reloc_idx=%u" - ", reloc_offset=%"PRIu64, j, r->submit_offset, r->or, r->shift, - r->reloc_idx, r->reloc_offset); - } - } -} - -static inline void get_abs_timeout(struct drm_msm_timespec *tv, uint64_t ns) -{ - struct timespec t; - uint32_t s = ns / 1000000000; - clock_gettime(CLOCK_MONOTONIC, &t); - tv->tv_sec = t.tv_sec + s; - tv->tv_nsec = t.tv_nsec + ns - (s * 1000000000); -} - -/* - * Stupid/simple growable array implementation: - */ - -static inline void * -grow(void *ptr, uint16_t nr, uint16_t *max, uint16_t sz) -{ - if ((nr + 1) > *max) { - if ((*max * 2) < (nr + 1)) - *max = nr + 5; - else - *max = *max * 2; - ptr = realloc(ptr, *max * sz); - } - return ptr; -} - -#define DECLARE_ARRAY(type, name) \ - unsigned short nr_ ## name, max_ ## name; \ - type * name; - -#define APPEND(x, name) ({ \ - (x)->name = grow((x)->name, (x)->nr_ ## name, &(x)->max_ ## name, sizeof((x)->name[0])); \ - (x)->nr_ ## name ++; \ -}) - -#endif /* MSM_PRIV_H_ */ diff --git a/lib/mesa/src/freedreno/drm/msm_ringbuffer.c b/lib/mesa/src/freedreno/drm/msm_ringbuffer.c deleted file mode 100644 index 369f26f98..000000000 --- a/lib/mesa/src/freedreno/drm/msm_ringbuffer.c +++ /dev/null @@ -1,722 +0,0 @@ -/* - * Copyright (C) 2012-2018 Rob Clark <robclark@freedesktop.org> - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * Authors: - * Rob Clark <robclark@freedesktop.org> - */ - -#include <assert.h> -#include <inttypes.h> - -#include "util/hash_table.h" -#include "util/set.h" -#include "util/slab.h" - -#include "drm/freedreno_ringbuffer.h" -#include "msm_priv.h" - -/* The legacy implementation of submit/ringbuffer, which still does the - * traditional reloc and cmd tracking - */ - - -#define INIT_SIZE 0x1000 - -static pthread_mutex_t idx_lock = PTHREAD_MUTEX_INITIALIZER; - - -struct msm_submit { - struct fd_submit base; - - DECLARE_ARRAY(struct drm_msm_gem_submit_bo, submit_bos); - DECLARE_ARRAY(struct fd_bo *, bos); - - unsigned seqno; - - /* maps fd_bo to idx in bos table: */ - struct hash_table *bo_table; - - struct slab_mempool ring_pool; - - /* hash-set of associated rings: */ - struct set *ring_set; - - struct fd_ringbuffer *primary; - - /* Allow for sub-allocation of stateobj ring buffers (ie. sharing - * the same underlying bo).. - * - * We also rely on previous stateobj having been fully constructed - * so we can reclaim extra space at it's end. - */ - struct fd_ringbuffer *suballoc_ring; -}; -FD_DEFINE_CAST(fd_submit, msm_submit); - -/* for FD_RINGBUFFER_GROWABLE rb's, tracks the 'finalized' cmdstream buffers - * and sizes. Ie. a finalized buffer can have no more commands appended to - * it. - */ -struct msm_cmd { - struct fd_bo *ring_bo; - unsigned size; - DECLARE_ARRAY(struct drm_msm_gem_submit_reloc, relocs); -}; - -static struct msm_cmd * -cmd_new(struct fd_bo *ring_bo) -{ - struct msm_cmd *cmd = malloc(sizeof(*cmd)); - cmd->ring_bo = fd_bo_ref(ring_bo); - cmd->size = 0; - cmd->nr_relocs = cmd->max_relocs = 0; - cmd->relocs = NULL; - return cmd; -} - -static void -cmd_free(struct msm_cmd *cmd) -{ - fd_bo_del(cmd->ring_bo); - free(cmd->relocs); - free(cmd); -} - -/* for _FD_RINGBUFFER_OBJECT rb's we need to track the bo's and flags to - * later copy into the submit when the stateobj rb is later referenced by - * a regular rb: - */ -struct msm_reloc_bo { - struct fd_bo *bo; - unsigned flags; -}; - -struct msm_ringbuffer { - struct fd_ringbuffer base; - - /* for FD_RINGBUFFER_STREAMING rb's which are sub-allocated */ - unsigned offset; - - union { - /* for _FD_RINGBUFFER_OBJECT case: */ - struct { - struct fd_pipe *pipe; - DECLARE_ARRAY(struct msm_reloc_bo, reloc_bos); - struct set *ring_set; - }; - /* for other cases: */ - struct { - struct fd_submit *submit; - DECLARE_ARRAY(struct msm_cmd *, cmds); - }; - } u; - - struct msm_cmd *cmd; /* current cmd */ - struct fd_bo *ring_bo; -}; -FD_DEFINE_CAST(fd_ringbuffer, msm_ringbuffer); - -static void finalize_current_cmd(struct fd_ringbuffer *ring); -static struct fd_ringbuffer * msm_ringbuffer_init( - struct msm_ringbuffer *msm_ring, - uint32_t size, enum fd_ringbuffer_flags flags); - -/* add (if needed) bo to submit and return index: */ -static uint32_t -append_bo(struct msm_submit *submit, struct fd_bo *bo, uint32_t flags) -{ - struct msm_bo *msm_bo = to_msm_bo(bo); - uint32_t idx; - pthread_mutex_lock(&idx_lock); - if (likely(msm_bo->current_submit_seqno == submit->seqno)) { - idx = msm_bo->idx; - } else { - uint32_t hash = _mesa_hash_pointer(bo); - struct hash_entry *entry; - - entry = _mesa_hash_table_search_pre_hashed(submit->bo_table, hash, bo); - if (entry) { - /* found */ - idx = (uint32_t)(uintptr_t)entry->data; - } else { - idx = APPEND(submit, submit_bos); - idx = APPEND(submit, bos); - - submit->submit_bos[idx].flags = 0; - submit->submit_bos[idx].handle = bo->handle; - submit->submit_bos[idx].presumed = 0; - - submit->bos[idx] = fd_bo_ref(bo); - - _mesa_hash_table_insert_pre_hashed(submit->bo_table, hash, bo, - (void *)(uintptr_t)idx); - } - msm_bo->current_submit_seqno = submit->seqno; - msm_bo->idx = idx; - } - pthread_mutex_unlock(&idx_lock); - if (flags & FD_RELOC_READ) - submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_READ; - if (flags & FD_RELOC_WRITE) - submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_WRITE; - return idx; -} - -static void -append_ring(struct set *set, struct fd_ringbuffer *ring) -{ - uint32_t hash = _mesa_hash_pointer(ring); - - if (!_mesa_set_search_pre_hashed(set, hash, ring)) { - fd_ringbuffer_ref(ring); - _mesa_set_add_pre_hashed(set, hash, ring); - } -} - -static void -msm_submit_suballoc_ring_bo(struct fd_submit *submit, - struct msm_ringbuffer *msm_ring, uint32_t size) -{ - struct msm_submit *msm_submit = to_msm_submit(submit); - unsigned suballoc_offset = 0; - struct fd_bo *suballoc_bo = NULL; - - if (msm_submit->suballoc_ring) { - struct msm_ringbuffer *suballoc_ring = - to_msm_ringbuffer(msm_submit->suballoc_ring); - - suballoc_bo = suballoc_ring->ring_bo; - suballoc_offset = fd_ringbuffer_size(msm_submit->suballoc_ring) + - suballoc_ring->offset; - - suballoc_offset = align(suballoc_offset, 0x10); - - if ((size + suballoc_offset) > suballoc_bo->size) { - suballoc_bo = NULL; - } - } - - if (!suballoc_bo) { - // TODO possibly larger size for streaming bo? - msm_ring->ring_bo = fd_bo_new_ring( - submit->pipe->dev, 0x8000, 0); - msm_ring->offset = 0; - } else { - msm_ring->ring_bo = fd_bo_ref(suballoc_bo); - msm_ring->offset = suballoc_offset; - } - - struct fd_ringbuffer *old_suballoc_ring = msm_submit->suballoc_ring; - - msm_submit->suballoc_ring = fd_ringbuffer_ref(&msm_ring->base); - - if (old_suballoc_ring) - fd_ringbuffer_del(old_suballoc_ring); -} - -static struct fd_ringbuffer * -msm_submit_new_ringbuffer(struct fd_submit *submit, uint32_t size, - enum fd_ringbuffer_flags flags) -{ - struct msm_submit *msm_submit = to_msm_submit(submit); - struct msm_ringbuffer *msm_ring; - - msm_ring = slab_alloc_st(&msm_submit->ring_pool); - - msm_ring->u.submit = submit; - - /* NOTE: needs to be before _suballoc_ring_bo() since it could - * increment the refcnt of the current ring - */ - msm_ring->base.refcnt = 1; - - if (flags & FD_RINGBUFFER_STREAMING) { - msm_submit_suballoc_ring_bo(submit, msm_ring, size); - } else { - if (flags & FD_RINGBUFFER_GROWABLE) - size = INIT_SIZE; - - msm_ring->offset = 0; - msm_ring->ring_bo = fd_bo_new_ring(submit->pipe->dev, size, 0); - } - - if (!msm_ringbuffer_init(msm_ring, size, flags)) - return NULL; - - if (flags & FD_RINGBUFFER_PRIMARY) { - debug_assert(!msm_submit->primary); - msm_submit->primary = fd_ringbuffer_ref(&msm_ring->base); - } - - return &msm_ring->base; -} - -static struct drm_msm_gem_submit_reloc * -handle_stateobj_relocs(struct msm_submit *submit, struct msm_ringbuffer *ring) -{ - struct msm_cmd *cmd = ring->cmd; - struct drm_msm_gem_submit_reloc *relocs; - - relocs = malloc(cmd->nr_relocs * sizeof(*relocs)); - - for (unsigned i = 0; i < cmd->nr_relocs; i++) { - unsigned idx = cmd->relocs[i].reloc_idx; - struct fd_bo *bo = ring->u.reloc_bos[idx].bo; - unsigned flags = 0; - - if (ring->u.reloc_bos[idx].flags & MSM_SUBMIT_BO_READ) - flags |= FD_RELOC_READ; - if (ring->u.reloc_bos[idx].flags & MSM_SUBMIT_BO_WRITE) - flags |= FD_RELOC_WRITE; - - relocs[i] = cmd->relocs[i]; - relocs[i].reloc_idx = append_bo(submit, bo, flags); - } - - return relocs; -} - -static int -msm_submit_flush(struct fd_submit *submit, int in_fence_fd, - int *out_fence_fd, uint32_t *out_fence) -{ - struct msm_submit *msm_submit = to_msm_submit(submit); - struct msm_pipe *msm_pipe = to_msm_pipe(submit->pipe); - struct drm_msm_gem_submit req = { - .flags = msm_pipe->pipe, - .queueid = msm_pipe->queue_id, - }; - int ret; - - debug_assert(msm_submit->primary); - - finalize_current_cmd(msm_submit->primary); - append_ring(msm_submit->ring_set, msm_submit->primary); - - unsigned nr_cmds = 0; - unsigned nr_objs = 0; - - set_foreach(msm_submit->ring_set, entry) { - struct fd_ringbuffer *ring = (void *)entry->key; - if (ring->flags & _FD_RINGBUFFER_OBJECT) { - nr_cmds += 1; - nr_objs += 1; - } else { - if (ring != msm_submit->primary) - finalize_current_cmd(ring); - nr_cmds += to_msm_ringbuffer(ring)->u.nr_cmds; - } - } - - void *obj_relocs[nr_objs]; - struct drm_msm_gem_submit_cmd cmds[nr_cmds]; - unsigned i = 0, o = 0; - - set_foreach(msm_submit->ring_set, entry) { - struct fd_ringbuffer *ring = (void *)entry->key; - struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); - - debug_assert(i < nr_cmds); - - // TODO handle relocs: - if (ring->flags & _FD_RINGBUFFER_OBJECT) { - - debug_assert(o < nr_objs); - - void *relocs = handle_stateobj_relocs(msm_submit, msm_ring); - obj_relocs[o++] = relocs; - - cmds[i].type = MSM_SUBMIT_CMD_IB_TARGET_BUF; - cmds[i].submit_idx = - append_bo(msm_submit, msm_ring->ring_bo, FD_RELOC_READ); - cmds[i].submit_offset = msm_ring->offset; - cmds[i].size = offset_bytes(ring->cur, ring->start); - cmds[i].pad = 0; - cmds[i].nr_relocs = msm_ring->cmd->nr_relocs; - cmds[i].relocs = VOID2U64(relocs); - - i++; - } else { - for (unsigned j = 0; j < msm_ring->u.nr_cmds; j++) { - if (ring->flags & FD_RINGBUFFER_PRIMARY) { - cmds[i].type = MSM_SUBMIT_CMD_BUF; - } else { - cmds[i].type = MSM_SUBMIT_CMD_IB_TARGET_BUF; - } - cmds[i].submit_idx = append_bo(msm_submit, - msm_ring->u.cmds[j]->ring_bo, FD_RELOC_READ); - cmds[i].submit_offset = msm_ring->offset; - cmds[i].size = msm_ring->u.cmds[j]->size; - cmds[i].pad = 0; - cmds[i].nr_relocs = msm_ring->u.cmds[j]->nr_relocs; - cmds[i].relocs = VOID2U64(msm_ring->u.cmds[j]->relocs); - - i++; - } - } - } - - if (in_fence_fd != -1) { - req.flags |= MSM_SUBMIT_FENCE_FD_IN | MSM_SUBMIT_NO_IMPLICIT; - req.fence_fd = in_fence_fd; - } - - if (out_fence_fd) { - req.flags |= MSM_SUBMIT_FENCE_FD_OUT; - } - - /* needs to be after get_cmd() as that could create bos/cmds table: */ - req.bos = VOID2U64(msm_submit->submit_bos), - req.nr_bos = msm_submit->nr_submit_bos; - req.cmds = VOID2U64(cmds), - req.nr_cmds = nr_cmds; - - DEBUG_MSG("nr_cmds=%u, nr_bos=%u", req.nr_cmds, req.nr_bos); - - ret = drmCommandWriteRead(submit->pipe->dev->fd, DRM_MSM_GEM_SUBMIT, - &req, sizeof(req)); - if (ret) { - ERROR_MSG("submit failed: %d (%s)", ret, strerror(errno)); - msm_dump_submit(&req); - } else if (!ret) { - if (out_fence) - *out_fence = req.fence; - - if (out_fence_fd) - *out_fence_fd = req.fence_fd; - } - - for (unsigned o = 0; o < nr_objs; o++) - free(obj_relocs[o]); - - return ret; -} - -static void -unref_rings(struct set_entry *entry) -{ - struct fd_ringbuffer *ring = (void *)entry->key; - fd_ringbuffer_del(ring); -} - -static void -msm_submit_destroy(struct fd_submit *submit) -{ - struct msm_submit *msm_submit = to_msm_submit(submit); - - if (msm_submit->primary) - fd_ringbuffer_del(msm_submit->primary); - if (msm_submit->suballoc_ring) - fd_ringbuffer_del(msm_submit->suballoc_ring); - - _mesa_hash_table_destroy(msm_submit->bo_table, NULL); - _mesa_set_destroy(msm_submit->ring_set, unref_rings); - - // TODO it would be nice to have a way to debug_assert() if all - // rb's haven't been free'd back to the slab, because that is - // an indication that we are leaking bo's - slab_destroy(&msm_submit->ring_pool); - - for (unsigned i = 0; i < msm_submit->nr_bos; i++) - fd_bo_del(msm_submit->bos[i]); - - free(msm_submit->submit_bos); - free(msm_submit->bos); - free(msm_submit); -} - -static const struct fd_submit_funcs submit_funcs = { - .new_ringbuffer = msm_submit_new_ringbuffer, - .flush = msm_submit_flush, - .destroy = msm_submit_destroy, -}; - -struct fd_submit * -msm_submit_new(struct fd_pipe *pipe) -{ - struct msm_submit *msm_submit = calloc(1, sizeof(*msm_submit)); - struct fd_submit *submit; - static unsigned submit_cnt = 0; - - msm_submit->seqno = ++submit_cnt; - msm_submit->bo_table = _mesa_hash_table_create(NULL, - _mesa_hash_pointer, _mesa_key_pointer_equal); - msm_submit->ring_set = _mesa_set_create(NULL, - _mesa_hash_pointer, _mesa_key_pointer_equal); - // TODO tune size: - slab_create(&msm_submit->ring_pool, sizeof(struct msm_ringbuffer), 16); - - submit = &msm_submit->base; - submit->pipe = pipe; - submit->funcs = &submit_funcs; - - return submit; -} - - -static void -finalize_current_cmd(struct fd_ringbuffer *ring) -{ - struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); - - debug_assert(!(ring->flags & _FD_RINGBUFFER_OBJECT)); - - if (!msm_ring->cmd) - return; - - debug_assert(msm_ring->cmd->ring_bo == msm_ring->ring_bo); - - unsigned idx = APPEND(&msm_ring->u, cmds); - - msm_ring->u.cmds[idx] = msm_ring->cmd; - msm_ring->cmd = NULL; - - msm_ring->u.cmds[idx]->size = offset_bytes(ring->cur, ring->start); -} - -static void -msm_ringbuffer_grow(struct fd_ringbuffer *ring, uint32_t size) -{ - struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); - struct fd_pipe *pipe = msm_ring->u.submit->pipe; - - debug_assert(ring->flags & FD_RINGBUFFER_GROWABLE); - - finalize_current_cmd(ring); - - fd_bo_del(msm_ring->ring_bo); - msm_ring->ring_bo = fd_bo_new_ring(pipe->dev, size, 0); - msm_ring->cmd = cmd_new(msm_ring->ring_bo); - - ring->start = fd_bo_map(msm_ring->ring_bo); - ring->end = &(ring->start[size/4]); - ring->cur = ring->start; - ring->size = size; -} - -static void -msm_ringbuffer_emit_reloc(struct fd_ringbuffer *ring, - const struct fd_reloc *reloc) -{ - struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); - struct fd_pipe *pipe; - unsigned reloc_idx; - - if (ring->flags & _FD_RINGBUFFER_OBJECT) { - unsigned idx = APPEND(&msm_ring->u, reloc_bos); - - msm_ring->u.reloc_bos[idx].bo = fd_bo_ref(reloc->bo); - msm_ring->u.reloc_bos[idx].flags = reloc->flags; - - /* this gets fixed up at submit->flush() time, since this state- - * object rb can be used with many different submits - */ - reloc_idx = idx; - - pipe = msm_ring->u.pipe; - } else { - struct msm_submit *msm_submit = - to_msm_submit(msm_ring->u.submit); - - reloc_idx = append_bo(msm_submit, reloc->bo, reloc->flags); - - pipe = msm_ring->u.submit->pipe; - } - - struct drm_msm_gem_submit_reloc *r; - unsigned idx = APPEND(msm_ring->cmd, relocs); - - r = &msm_ring->cmd->relocs[idx]; - - r->reloc_idx = reloc_idx; - r->reloc_offset = reloc->offset; - r->or = reloc->or; - r->shift = reloc->shift; - r->submit_offset = offset_bytes(ring->cur, ring->start) + - msm_ring->offset; - - ring->cur++; - - if (pipe->gpu_id >= 500) { - idx = APPEND(msm_ring->cmd, relocs); - r = &msm_ring->cmd->relocs[idx]; - - r->reloc_idx = reloc_idx; - r->reloc_offset = reloc->offset; - r->or = reloc->orhi; - r->shift = reloc->shift - 32; - r->submit_offset = offset_bytes(ring->cur, ring->start) + - msm_ring->offset; - - ring->cur++; - } -} - -static void -append_stateobj_rings(struct msm_submit *submit, struct fd_ringbuffer *target) -{ - struct msm_ringbuffer *msm_target = to_msm_ringbuffer(target); - - debug_assert(target->flags & _FD_RINGBUFFER_OBJECT); - - set_foreach(msm_target->u.ring_set, entry) { - struct fd_ringbuffer *ring = (void *)entry->key; - - append_ring(submit->ring_set, ring); - - if (ring->flags & _FD_RINGBUFFER_OBJECT) { - append_stateobj_rings(submit, ring); - } - } -} - -static uint32_t -msm_ringbuffer_emit_reloc_ring(struct fd_ringbuffer *ring, - struct fd_ringbuffer *target, uint32_t cmd_idx) -{ - struct msm_ringbuffer *msm_target = to_msm_ringbuffer(target); - struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); - struct fd_bo *bo; - uint32_t size; - - if ((target->flags & FD_RINGBUFFER_GROWABLE) && - (cmd_idx < msm_target->u.nr_cmds)) { - bo = msm_target->u.cmds[cmd_idx]->ring_bo; - size = msm_target->u.cmds[cmd_idx]->size; - } else { - bo = msm_target->ring_bo; - size = offset_bytes(target->cur, target->start); - } - - msm_ringbuffer_emit_reloc(ring, &(struct fd_reloc){ - .bo = bo, - .flags = FD_RELOC_READ, - .offset = msm_target->offset, - }); - - if ((target->flags & _FD_RINGBUFFER_OBJECT) && - !(ring->flags & _FD_RINGBUFFER_OBJECT)) { - struct msm_submit *msm_submit = to_msm_submit(msm_ring->u.submit); - - append_stateobj_rings(msm_submit, target); - } - - if (ring->flags & _FD_RINGBUFFER_OBJECT) { - append_ring(msm_ring->u.ring_set, target); - } else { - struct msm_submit *msm_submit = to_msm_submit(msm_ring->u.submit); - append_ring(msm_submit->ring_set, target); - } - - return size; -} - -static uint32_t -msm_ringbuffer_cmd_count(struct fd_ringbuffer *ring) -{ - if (ring->flags & FD_RINGBUFFER_GROWABLE) - return to_msm_ringbuffer(ring)->u.nr_cmds + 1; - return 1; -} - -static void -msm_ringbuffer_destroy(struct fd_ringbuffer *ring) -{ - struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); - - fd_bo_del(msm_ring->ring_bo); - if (msm_ring->cmd) - cmd_free(msm_ring->cmd); - - if (ring->flags & _FD_RINGBUFFER_OBJECT) { - for (unsigned i = 0; i < msm_ring->u.nr_reloc_bos; i++) { - fd_bo_del(msm_ring->u.reloc_bos[i].bo); - } - - _mesa_set_destroy(msm_ring->u.ring_set, unref_rings); - - free(msm_ring->u.reloc_bos); - free(msm_ring); - } else { - struct fd_submit *submit = msm_ring->u.submit; - - for (unsigned i = 0; i < msm_ring->u.nr_cmds; i++) { - cmd_free(msm_ring->u.cmds[i]); - } - - free(msm_ring->u.cmds); - slab_free_st(&to_msm_submit(submit)->ring_pool, msm_ring); - } -} - -static const struct fd_ringbuffer_funcs ring_funcs = { - .grow = msm_ringbuffer_grow, - .emit_reloc = msm_ringbuffer_emit_reloc, - .emit_reloc_ring = msm_ringbuffer_emit_reloc_ring, - .cmd_count = msm_ringbuffer_cmd_count, - .destroy = msm_ringbuffer_destroy, -}; - -static inline struct fd_ringbuffer * -msm_ringbuffer_init(struct msm_ringbuffer *msm_ring, uint32_t size, - enum fd_ringbuffer_flags flags) -{ - struct fd_ringbuffer *ring = &msm_ring->base; - - debug_assert(msm_ring->ring_bo); - - uint8_t *base = fd_bo_map(msm_ring->ring_bo); - ring->start = (void *)(base + msm_ring->offset); - ring->end = &(ring->start[size/4]); - ring->cur = ring->start; - - ring->size = size; - ring->flags = flags; - - ring->funcs = &ring_funcs; - - msm_ring->u.cmds = NULL; - msm_ring->u.nr_cmds = msm_ring->u.max_cmds = 0; - - msm_ring->cmd = cmd_new(msm_ring->ring_bo); - - return ring; -} - -struct fd_ringbuffer * -msm_ringbuffer_new_object(struct fd_pipe *pipe, uint32_t size) -{ - struct msm_ringbuffer *msm_ring = malloc(sizeof(*msm_ring)); - - msm_ring->u.pipe = pipe; - msm_ring->offset = 0; - msm_ring->ring_bo = fd_bo_new_ring(pipe->dev, size, 0); - msm_ring->base.refcnt = 1; - - msm_ring->u.reloc_bos = NULL; - msm_ring->u.nr_reloc_bos = msm_ring->u.max_reloc_bos = 0; - - msm_ring->u.ring_set = _mesa_set_create(NULL, - _mesa_hash_pointer, _mesa_key_pointer_equal); - - return msm_ringbuffer_init(msm_ring, size, _FD_RINGBUFFER_OBJECT); -} diff --git a/lib/mesa/src/freedreno/drm/msm_ringbuffer_sp.c b/lib/mesa/src/freedreno/drm/msm_ringbuffer_sp.c deleted file mode 100644 index 2b8f53172..000000000 --- a/lib/mesa/src/freedreno/drm/msm_ringbuffer_sp.c +++ /dev/null @@ -1,568 +0,0 @@ -/* - * Copyright (C) 2018 Rob Clark <robclark@freedesktop.org> - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * Authors: - * Rob Clark <robclark@freedesktop.org> - */ - -#include <assert.h> -#include <inttypes.h> - -#include "util/hash_table.h" -#include "util/slab.h" - -#include "drm/freedreno_ringbuffer.h" -#include "msm_priv.h" - -/* A "softpin" implementation of submit/ringbuffer, which lowers CPU overhead - * by avoiding the additional tracking necessary to build cmds/relocs tables - * (but still builds a bos table) - */ - - -#define INIT_SIZE 0x1000 - -static pthread_mutex_t idx_lock = PTHREAD_MUTEX_INITIALIZER; - - -struct msm_submit_sp { - struct fd_submit base; - - DECLARE_ARRAY(struct drm_msm_gem_submit_bo, submit_bos); - DECLARE_ARRAY(struct fd_bo *, bos); - - unsigned seqno; - - /* maps fd_bo to idx in bos table: */ - struct hash_table *bo_table; - - struct slab_mempool ring_pool; - - struct fd_ringbuffer *primary; - - /* Allow for sub-allocation of stateobj ring buffers (ie. sharing - * the same underlying bo).. - * - * We also rely on previous stateobj having been fully constructed - * so we can reclaim extra space at it's end. - */ - struct fd_ringbuffer *suballoc_ring; -}; -FD_DEFINE_CAST(fd_submit, msm_submit_sp); - -/* for FD_RINGBUFFER_GROWABLE rb's, tracks the 'finalized' cmdstream buffers - * and sizes. Ie. a finalized buffer can have no more commands appended to - * it. - */ -struct msm_cmd_sp { - struct fd_bo *ring_bo; - unsigned size; -}; - -/* for _FD_RINGBUFFER_OBJECT rb's we need to track the bo's and flags to - * later copy into the submit when the stateobj rb is later referenced by - * a regular rb: - */ -struct msm_reloc_bo_sp { - struct fd_bo *bo; - unsigned flags; -}; - -struct msm_ringbuffer_sp { - struct fd_ringbuffer base; - - /* for FD_RINGBUFFER_STREAMING rb's which are sub-allocated */ - unsigned offset; - -// TODO check disasm.. hopefully compilers CSE can realize that -// reloc_bos and cmds are at the same offsets and optimize some -// divergent cases into single case - union { - /* for _FD_RINGBUFFER_OBJECT case: */ - struct { - struct fd_pipe *pipe; - DECLARE_ARRAY(struct msm_reloc_bo_sp, reloc_bos); - }; - /* for other cases: */ - struct { - struct fd_submit *submit; - DECLARE_ARRAY(struct msm_cmd_sp, cmds); - }; - } u; - - struct fd_bo *ring_bo; -}; -FD_DEFINE_CAST(fd_ringbuffer, msm_ringbuffer_sp); - -static void finalize_current_cmd(struct fd_ringbuffer *ring); -static struct fd_ringbuffer * msm_ringbuffer_sp_init( - struct msm_ringbuffer_sp *msm_ring, - uint32_t size, enum fd_ringbuffer_flags flags); - -/* add (if needed) bo to submit and return index: */ -static uint32_t -append_bo(struct msm_submit_sp *submit, struct fd_bo *bo, uint32_t flags) -{ - struct msm_bo *msm_bo = to_msm_bo(bo); - uint32_t idx; - pthread_mutex_lock(&idx_lock); - if (likely(msm_bo->current_submit_seqno == submit->seqno)) { - idx = msm_bo->idx; - } else { - uint32_t hash = _mesa_hash_pointer(bo); - struct hash_entry *entry; - - entry = _mesa_hash_table_search_pre_hashed(submit->bo_table, hash, bo); - if (entry) { - /* found */ - idx = (uint32_t)(uintptr_t)entry->data; - } else { - idx = APPEND(submit, submit_bos); - idx = APPEND(submit, bos); - - submit->submit_bos[idx].flags = 0; - submit->submit_bos[idx].handle = bo->handle; - submit->submit_bos[idx].presumed = 0; - - submit->bos[idx] = fd_bo_ref(bo); - - _mesa_hash_table_insert_pre_hashed(submit->bo_table, hash, bo, - (void *)(uintptr_t)idx); - } - msm_bo->current_submit_seqno = submit->seqno; - msm_bo->idx = idx; - } - pthread_mutex_unlock(&idx_lock); - if (flags & FD_RELOC_READ) - submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_READ; - if (flags & FD_RELOC_WRITE) - submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_WRITE; - if (flags & FD_RELOC_DUMP) - submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_DUMP; - return idx; -} - -static void -msm_submit_suballoc_ring_bo(struct fd_submit *submit, - struct msm_ringbuffer_sp *msm_ring, uint32_t size) -{ - struct msm_submit_sp *msm_submit = to_msm_submit_sp(submit); - unsigned suballoc_offset = 0; - struct fd_bo *suballoc_bo = NULL; - - if (msm_submit->suballoc_ring) { - struct msm_ringbuffer_sp *suballoc_ring = - to_msm_ringbuffer_sp(msm_submit->suballoc_ring); - - suballoc_bo = suballoc_ring->ring_bo; - suballoc_offset = fd_ringbuffer_size(msm_submit->suballoc_ring) + - suballoc_ring->offset; - - suballoc_offset = align(suballoc_offset, 0x10); - - if ((size + suballoc_offset) > suballoc_bo->size) { - suballoc_bo = NULL; - } - } - - if (!suballoc_bo) { - // TODO possibly larger size for streaming bo? - msm_ring->ring_bo = fd_bo_new_ring(submit->pipe->dev, - 0x8000, DRM_FREEDRENO_GEM_GPUREADONLY); - msm_ring->offset = 0; - } else { - msm_ring->ring_bo = fd_bo_ref(suballoc_bo); - msm_ring->offset = suballoc_offset; - } - - struct fd_ringbuffer *old_suballoc_ring = msm_submit->suballoc_ring; - - msm_submit->suballoc_ring = fd_ringbuffer_ref(&msm_ring->base); - - if (old_suballoc_ring) - fd_ringbuffer_del(old_suballoc_ring); -} - -static struct fd_ringbuffer * -msm_submit_sp_new_ringbuffer(struct fd_submit *submit, uint32_t size, - enum fd_ringbuffer_flags flags) -{ - struct msm_submit_sp *msm_submit = to_msm_submit_sp(submit); - struct msm_ringbuffer_sp *msm_ring; - - msm_ring = slab_alloc_st(&msm_submit->ring_pool); - - msm_ring->u.submit = submit; - - /* NOTE: needs to be before _suballoc_ring_bo() since it could - * increment the refcnt of the current ring - */ - msm_ring->base.refcnt = 1; - - if (flags & FD_RINGBUFFER_STREAMING) { - msm_submit_suballoc_ring_bo(submit, msm_ring, size); - } else { - if (flags & FD_RINGBUFFER_GROWABLE) - size = INIT_SIZE; - - msm_ring->offset = 0; - msm_ring->ring_bo = fd_bo_new_ring(submit->pipe->dev, size, - DRM_FREEDRENO_GEM_GPUREADONLY); - } - - if (!msm_ringbuffer_sp_init(msm_ring, size, flags)) - return NULL; - - if (flags & FD_RINGBUFFER_PRIMARY) { - debug_assert(!msm_submit->primary); - msm_submit->primary = fd_ringbuffer_ref(&msm_ring->base); - } - - return &msm_ring->base; -} - -static int -msm_submit_sp_flush(struct fd_submit *submit, int in_fence_fd, - int *out_fence_fd, uint32_t *out_fence) -{ - struct msm_submit_sp *msm_submit = to_msm_submit_sp(submit); - struct msm_pipe *msm_pipe = to_msm_pipe(submit->pipe); - struct drm_msm_gem_submit req = { - .flags = msm_pipe->pipe, - .queueid = msm_pipe->queue_id, - }; - int ret; - - debug_assert(msm_submit->primary); - finalize_current_cmd(msm_submit->primary); - - struct msm_ringbuffer_sp *primary = to_msm_ringbuffer_sp(msm_submit->primary); - struct drm_msm_gem_submit_cmd cmds[primary->u.nr_cmds]; - - for (unsigned i = 0; i < primary->u.nr_cmds; i++) { - cmds[i].type = MSM_SUBMIT_CMD_BUF; - cmds[i].submit_idx = append_bo(msm_submit, - primary->u.cmds[i].ring_bo, FD_RELOC_READ | FD_RELOC_DUMP); - cmds[i].submit_offset = primary->offset; - cmds[i].size = primary->u.cmds[i].size; - cmds[i].pad = 0; - cmds[i].nr_relocs = 0; - } - - if (in_fence_fd != -1) { - req.flags |= MSM_SUBMIT_FENCE_FD_IN | MSM_SUBMIT_NO_IMPLICIT; - req.fence_fd = in_fence_fd; - } - - if (out_fence_fd) { - req.flags |= MSM_SUBMIT_FENCE_FD_OUT; - } - - /* needs to be after get_cmd() as that could create bos/cmds table: */ - req.bos = VOID2U64(msm_submit->submit_bos), - req.nr_bos = msm_submit->nr_submit_bos; - req.cmds = VOID2U64(cmds), - req.nr_cmds = primary->u.nr_cmds; - - DEBUG_MSG("nr_cmds=%u, nr_bos=%u", req.nr_cmds, req.nr_bos); - - ret = drmCommandWriteRead(submit->pipe->dev->fd, DRM_MSM_GEM_SUBMIT, - &req, sizeof(req)); - if (ret) { - ERROR_MSG("submit failed: %d (%s)", ret, strerror(errno)); - msm_dump_submit(&req); - } else if (!ret) { - if (out_fence) - *out_fence = req.fence; - - if (out_fence_fd) - *out_fence_fd = req.fence_fd; - } - - return ret; -} - -static void -msm_submit_sp_destroy(struct fd_submit *submit) -{ - struct msm_submit_sp *msm_submit = to_msm_submit_sp(submit); - - if (msm_submit->primary) - fd_ringbuffer_del(msm_submit->primary); - if (msm_submit->suballoc_ring) - fd_ringbuffer_del(msm_submit->suballoc_ring); - - _mesa_hash_table_destroy(msm_submit->bo_table, NULL); - - // TODO it would be nice to have a way to debug_assert() if all - // rb's haven't been free'd back to the slab, because that is - // an indication that we are leaking bo's - slab_destroy(&msm_submit->ring_pool); - - for (unsigned i = 0; i < msm_submit->nr_bos; i++) - fd_bo_del(msm_submit->bos[i]); - - free(msm_submit->submit_bos); - free(msm_submit->bos); - free(msm_submit); -} - -static const struct fd_submit_funcs submit_funcs = { - .new_ringbuffer = msm_submit_sp_new_ringbuffer, - .flush = msm_submit_sp_flush, - .destroy = msm_submit_sp_destroy, -}; - -struct fd_submit * -msm_submit_sp_new(struct fd_pipe *pipe) -{ - struct msm_submit_sp *msm_submit = calloc(1, sizeof(*msm_submit)); - struct fd_submit *submit; - static unsigned submit_cnt = 0; - - msm_submit->seqno = ++submit_cnt; - msm_submit->bo_table = _mesa_hash_table_create(NULL, - _mesa_hash_pointer, _mesa_key_pointer_equal); - // TODO tune size: - slab_create(&msm_submit->ring_pool, sizeof(struct msm_ringbuffer_sp), 16); - - submit = &msm_submit->base; - submit->pipe = pipe; - submit->funcs = &submit_funcs; - - return submit; -} - - -static void -finalize_current_cmd(struct fd_ringbuffer *ring) -{ - debug_assert(!(ring->flags & _FD_RINGBUFFER_OBJECT)); - - struct msm_ringbuffer_sp *msm_ring = to_msm_ringbuffer_sp(ring); - unsigned idx = APPEND(&msm_ring->u, cmds); - - msm_ring->u.cmds[idx].ring_bo = fd_bo_ref(msm_ring->ring_bo); - msm_ring->u.cmds[idx].size = offset_bytes(ring->cur, ring->start); -} - -static void -msm_ringbuffer_sp_grow(struct fd_ringbuffer *ring, uint32_t size) -{ - struct msm_ringbuffer_sp *msm_ring = to_msm_ringbuffer_sp(ring); - struct fd_pipe *pipe = msm_ring->u.submit->pipe; - - debug_assert(ring->flags & FD_RINGBUFFER_GROWABLE); - - finalize_current_cmd(ring); - - fd_bo_del(msm_ring->ring_bo); - msm_ring->ring_bo = fd_bo_new_ring(pipe->dev, size, - DRM_FREEDRENO_GEM_GPUREADONLY); - - ring->start = fd_bo_map(msm_ring->ring_bo); - ring->end = &(ring->start[size/4]); - ring->cur = ring->start; - ring->size = size; -} - -static void -msm_ringbuffer_sp_emit_reloc(struct fd_ringbuffer *ring, - const struct fd_reloc *reloc) -{ - struct msm_ringbuffer_sp *msm_ring = to_msm_ringbuffer_sp(ring); - struct fd_pipe *pipe; - - if (ring->flags & _FD_RINGBUFFER_OBJECT) { - unsigned idx = APPEND(&msm_ring->u, reloc_bos); - - msm_ring->u.reloc_bos[idx].bo = fd_bo_ref(reloc->bo); - msm_ring->u.reloc_bos[idx].flags = reloc->flags; - - pipe = msm_ring->u.pipe; - } else { - struct msm_submit_sp *msm_submit = - to_msm_submit_sp(msm_ring->u.submit); - - append_bo(msm_submit, reloc->bo, reloc->flags); - - pipe = msm_ring->u.submit->pipe; - } - - uint64_t iova = fd_bo_get_iova(reloc->bo) + reloc->offset; - uint32_t dword = iova; - int shift = reloc->shift; - - if (shift < 0) - dword >>= -shift; - else - dword <<= shift; - - (*ring->cur++) = dword | reloc->or; - - if (pipe->gpu_id >= 500) { - dword = iova >> 32; - shift -= 32; - - if (shift < 0) - dword >>= -shift; - else - dword <<= shift; - - (*ring->cur++) = dword | reloc->orhi; - } -} - -static uint32_t -msm_ringbuffer_sp_emit_reloc_ring(struct fd_ringbuffer *ring, - struct fd_ringbuffer *target, uint32_t cmd_idx) -{ - struct msm_ringbuffer_sp *msm_target = to_msm_ringbuffer_sp(target); - struct fd_bo *bo; - uint32_t size; - - if ((target->flags & FD_RINGBUFFER_GROWABLE) && - (cmd_idx < msm_target->u.nr_cmds)) { - bo = msm_target->u.cmds[cmd_idx].ring_bo; - size = msm_target->u.cmds[cmd_idx].size; - } else { - bo = msm_target->ring_bo; - size = offset_bytes(target->cur, target->start); - } - - msm_ringbuffer_sp_emit_reloc(ring, &(struct fd_reloc){ - .bo = bo, - .flags = FD_RELOC_READ | FD_RELOC_DUMP, - .offset = msm_target->offset, - }); - - if (!(target->flags & _FD_RINGBUFFER_OBJECT)) - return size; - - struct msm_ringbuffer_sp *msm_ring = to_msm_ringbuffer_sp(ring); - - if (ring->flags & _FD_RINGBUFFER_OBJECT) { - for (unsigned i = 0; i < msm_target->u.nr_reloc_bos; i++) { - unsigned idx = APPEND(&msm_ring->u, reloc_bos); - - msm_ring->u.reloc_bos[idx].bo = - fd_bo_ref(msm_target->u.reloc_bos[i].bo); - msm_ring->u.reloc_bos[idx].flags = - msm_target->u.reloc_bos[i].flags; - } - } else { - // TODO it would be nice to know whether we have already - // seen this target before. But hopefully we hit the - // append_bo() fast path enough for this to not matter: - struct msm_submit_sp *msm_submit = to_msm_submit_sp(msm_ring->u.submit); - - for (unsigned i = 0; i < msm_target->u.nr_reloc_bos; i++) { - append_bo(msm_submit, msm_target->u.reloc_bos[i].bo, - msm_target->u.reloc_bos[i].flags); - } - } - - return size; -} - -static uint32_t -msm_ringbuffer_sp_cmd_count(struct fd_ringbuffer *ring) -{ - if (ring->flags & FD_RINGBUFFER_GROWABLE) - return to_msm_ringbuffer_sp(ring)->u.nr_cmds + 1; - return 1; -} - -static void -msm_ringbuffer_sp_destroy(struct fd_ringbuffer *ring) -{ - struct msm_ringbuffer_sp *msm_ring = to_msm_ringbuffer_sp(ring); - - fd_bo_del(msm_ring->ring_bo); - - if (ring->flags & _FD_RINGBUFFER_OBJECT) { - for (unsigned i = 0; i < msm_ring->u.nr_reloc_bos; i++) { - fd_bo_del(msm_ring->u.reloc_bos[i].bo); - } - - free(msm_ring); - } else { - struct fd_submit *submit = msm_ring->u.submit; - - for (unsigned i = 0; i < msm_ring->u.nr_cmds; i++) { - fd_bo_del(msm_ring->u.cmds[i].ring_bo); - } - - slab_free_st(&to_msm_submit_sp(submit)->ring_pool, msm_ring); - } -} - -static const struct fd_ringbuffer_funcs ring_funcs = { - .grow = msm_ringbuffer_sp_grow, - .emit_reloc = msm_ringbuffer_sp_emit_reloc, - .emit_reloc_ring = msm_ringbuffer_sp_emit_reloc_ring, - .cmd_count = msm_ringbuffer_sp_cmd_count, - .destroy = msm_ringbuffer_sp_destroy, -}; - -static inline struct fd_ringbuffer * -msm_ringbuffer_sp_init(struct msm_ringbuffer_sp *msm_ring, uint32_t size, - enum fd_ringbuffer_flags flags) -{ - struct fd_ringbuffer *ring = &msm_ring->base; - - debug_assert(msm_ring->ring_bo); - - uint8_t *base = fd_bo_map(msm_ring->ring_bo); - ring->start = (void *)(base + msm_ring->offset); - ring->end = &(ring->start[size/4]); - ring->cur = ring->start; - - ring->size = size; - ring->flags = flags; - - ring->funcs = &ring_funcs; - - // TODO initializing these could probably be conditional on flags - // since unneed for FD_RINGBUFFER_STAGING case.. - msm_ring->u.cmds = NULL; - msm_ring->u.nr_cmds = msm_ring->u.max_cmds = 0; - - msm_ring->u.reloc_bos = NULL; - msm_ring->u.nr_reloc_bos = msm_ring->u.max_reloc_bos = 0; - - return ring; -} - -struct fd_ringbuffer * -msm_ringbuffer_sp_new_object(struct fd_pipe *pipe, uint32_t size) -{ - struct msm_ringbuffer_sp *msm_ring = malloc(sizeof(*msm_ring)); - - msm_ring->u.pipe = pipe; - msm_ring->offset = 0; - msm_ring->ring_bo = fd_bo_new_ring(pipe->dev, size, - DRM_FREEDRENO_GEM_GPUREADONLY); - msm_ring->base.refcnt = 1; - - return msm_ringbuffer_sp_init(msm_ring, size, _FD_RINGBUFFER_OBJECT); -} diff --git a/lib/mesa/src/freedreno/drm/msm_ringbuffer_sp.h b/lib/mesa/src/freedreno/drm/msm_ringbuffer_sp.h deleted file mode 100644 index 79dba3b84..000000000 --- a/lib/mesa/src/freedreno/drm/msm_ringbuffer_sp.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright © 2021 Google, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifdef X -# undef X -#endif - -#if PTRSZ == 32 -# define X(n) n ## _32 -#else -# define X(n) n ## _64 -#endif - - -static void -X(emit_reloc_common)(struct fd_ringbuffer *ring, const struct fd_reloc *reloc) -{ - (*ring->cur++) = (uint32_t)reloc->iova; -#if PTRSZ == 64 - (*ring->cur++) = (uint32_t)(reloc->iova >> 32); -#endif -} - -static void -X(msm_ringbuffer_sp_emit_reloc_nonobj)(struct fd_ringbuffer *ring, - const struct fd_reloc *reloc) -{ - X(emit_reloc_common)(ring, reloc); - - assert(!(ring->flags & _FD_RINGBUFFER_OBJECT)); - - struct msm_ringbuffer_sp *msm_ring = to_msm_ringbuffer_sp(ring); - - struct msm_submit_sp *msm_submit = - to_msm_submit_sp(msm_ring->u.submit); - - msm_submit_append_bo(msm_submit, reloc->bo); -} - -static void -X(msm_ringbuffer_sp_emit_reloc_obj)(struct fd_ringbuffer *ring, - const struct fd_reloc *reloc) -{ - X(emit_reloc_common)(ring, reloc); - - assert(ring->flags & _FD_RINGBUFFER_OBJECT); - - struct msm_ringbuffer_sp *msm_ring = to_msm_ringbuffer_sp(ring); - - /* Avoid emitting duplicate BO references into the list. Ringbuffer - * objects are long-lived, so this saves ongoing work at draw time in - * exchange for a bit at context setup/first draw. And the number of - * relocs per ringbuffer object is fairly small, so the O(n^2) doesn't - * hurt much. - */ - bool found = false; - for (int i = 0; i < msm_ring->u.nr_reloc_bos; i++) { - if (msm_ring->u.reloc_bos[i] == reloc->bo) { - found = true; - break; - } - } - if (!found) { - APPEND(&msm_ring->u, reloc_bos, fd_bo_ref(reloc->bo)); - } -} - -static uint32_t -X(msm_ringbuffer_sp_emit_reloc_ring)(struct fd_ringbuffer *ring, - struct fd_ringbuffer *target, uint32_t cmd_idx) -{ - struct msm_ringbuffer_sp *msm_target = to_msm_ringbuffer_sp(target); - struct fd_bo *bo; - uint32_t size; - - if ((target->flags & FD_RINGBUFFER_GROWABLE) && - (cmd_idx < msm_target->u.nr_cmds)) { - bo = msm_target->u.cmds[cmd_idx].ring_bo; - size = msm_target->u.cmds[cmd_idx].size; - } else { - bo = msm_target->ring_bo; - size = offset_bytes(target->cur, target->start); - } - - if (ring->flags & _FD_RINGBUFFER_OBJECT) { - X(msm_ringbuffer_sp_emit_reloc_obj)(ring, &(struct fd_reloc){ - .bo = bo, - .iova = bo->iova + msm_target->offset, - .offset = msm_target->offset, - }); - } else { - X(msm_ringbuffer_sp_emit_reloc_nonobj)(ring, &(struct fd_reloc){ - .bo = bo, - .iova = bo->iova + msm_target->offset, - .offset = msm_target->offset, - }); - } - - if (!(target->flags & _FD_RINGBUFFER_OBJECT)) - return size; - - struct msm_ringbuffer_sp *msm_ring = to_msm_ringbuffer_sp(ring); - - if (ring->flags & _FD_RINGBUFFER_OBJECT) { - for (unsigned i = 0; i < msm_target->u.nr_reloc_bos; i++) { - APPEND(&msm_ring->u, reloc_bos, fd_bo_ref(msm_target->u.reloc_bos[i])); - } - } else { - // TODO it would be nice to know whether we have already - // seen this target before. But hopefully we hit the - // append_bo() fast path enough for this to not matter: - struct msm_submit_sp *msm_submit = to_msm_submit_sp(msm_ring->u.submit); - - for (unsigned i = 0; i < msm_target->u.nr_reloc_bos; i++) { - msm_submit_append_bo(msm_submit, msm_target->u.reloc_bos[i]); - } - } - - return size; -} |