![Lucas Stach](/assets/img/avatar_default.png)
There are 3 big benefits to suballocating a single big DMA buffer for command submission: 1. Avoid hammering CMA. The old way of allocating and freeing a DMA buffer for each submission was hitting some of the real slow pathes in CMA, as this allocator was not designed for a concurrent small buffers load. 2. Less TLB flushes on IOMMUv2. If a new command buffer is mapped into the GPU address space the MMU TLBs need to be flushed. By having one big buffer statically mapped to the GPU, a lot of those flushes can be avoided. 3. No funky workarounds for GC3000. The FE TLB flush on GC3000 isn't reliable. To work around that we tried to lay out the cmdbufs in the GPU address space in a way to avoid this issue. This hasn't always worked if the address space is crowded. A single statically mapped buffer avoids the erratum completely. Signed-off-by: Lucas Stach <l.stach@pengutronix.de> Reviewed-by: Christian Gmeiner <christian.gmeiner@gmail.com>
437 lines
10 KiB
C
437 lines
10 KiB
C
/*
|
|
* Copyright (C) 2015 Etnaviv Project
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
* under the terms of the GNU General Public License version 2 as published by
|
|
* the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful, but WITHOUT
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
* more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License along with
|
|
* this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#include <linux/reservation.h>
|
|
#include "etnaviv_cmdbuf.h"
|
|
#include "etnaviv_drv.h"
|
|
#include "etnaviv_gpu.h"
|
|
#include "etnaviv_gem.h"
|
|
|
|
/*
|
|
* Cmdstream submission:
|
|
*/
|
|
|
|
#define BO_INVALID_FLAGS ~(ETNA_SUBMIT_BO_READ | ETNA_SUBMIT_BO_WRITE)
|
|
/* make sure these don't conflict w/ ETNAVIV_SUBMIT_BO_x */
|
|
#define BO_LOCKED 0x4000
|
|
#define BO_PINNED 0x2000
|
|
|
|
static struct etnaviv_gem_submit *submit_create(struct drm_device *dev,
|
|
struct etnaviv_gpu *gpu, size_t nr)
|
|
{
|
|
struct etnaviv_gem_submit *submit;
|
|
size_t sz = size_vstruct(nr, sizeof(submit->bos[0]), sizeof(*submit));
|
|
|
|
submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
|
|
if (submit) {
|
|
submit->dev = dev;
|
|
submit->gpu = gpu;
|
|
|
|
/* initially, until copy_from_user() and bo lookup succeeds: */
|
|
submit->nr_bos = 0;
|
|
|
|
ww_acquire_init(&submit->ticket, &reservation_ww_class);
|
|
}
|
|
|
|
return submit;
|
|
}
|
|
|
|
static int submit_lookup_objects(struct etnaviv_gem_submit *submit,
|
|
struct drm_file *file, struct drm_etnaviv_gem_submit_bo *submit_bos,
|
|
unsigned nr_bos)
|
|
{
|
|
struct drm_etnaviv_gem_submit_bo *bo;
|
|
unsigned i;
|
|
int ret = 0;
|
|
|
|
spin_lock(&file->table_lock);
|
|
|
|
for (i = 0, bo = submit_bos; i < nr_bos; i++, bo++) {
|
|
struct drm_gem_object *obj;
|
|
|
|
if (bo->flags & BO_INVALID_FLAGS) {
|
|
DRM_ERROR("invalid flags: %x\n", bo->flags);
|
|
ret = -EINVAL;
|
|
goto out_unlock;
|
|
}
|
|
|
|
submit->bos[i].flags = bo->flags;
|
|
|
|
/* normally use drm_gem_object_lookup(), but for bulk lookup
|
|
* all under single table_lock just hit object_idr directly:
|
|
*/
|
|
obj = idr_find(&file->object_idr, bo->handle);
|
|
if (!obj) {
|
|
DRM_ERROR("invalid handle %u at index %u\n",
|
|
bo->handle, i);
|
|
ret = -EINVAL;
|
|
goto out_unlock;
|
|
}
|
|
|
|
/*
|
|
* Take a refcount on the object. The file table lock
|
|
* prevents the object_idr's refcount on this being dropped.
|
|
*/
|
|
drm_gem_object_reference(obj);
|
|
|
|
submit->bos[i].obj = to_etnaviv_bo(obj);
|
|
}
|
|
|
|
out_unlock:
|
|
submit->nr_bos = i;
|
|
spin_unlock(&file->table_lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void submit_unlock_object(struct etnaviv_gem_submit *submit, int i)
|
|
{
|
|
if (submit->bos[i].flags & BO_LOCKED) {
|
|
struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
|
|
|
|
ww_mutex_unlock(&etnaviv_obj->resv->lock);
|
|
submit->bos[i].flags &= ~BO_LOCKED;
|
|
}
|
|
}
|
|
|
|
static int submit_lock_objects(struct etnaviv_gem_submit *submit)
|
|
{
|
|
int contended, slow_locked = -1, i, ret = 0;
|
|
|
|
retry:
|
|
for (i = 0; i < submit->nr_bos; i++) {
|
|
struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
|
|
|
|
if (slow_locked == i)
|
|
slow_locked = -1;
|
|
|
|
contended = i;
|
|
|
|
if (!(submit->bos[i].flags & BO_LOCKED)) {
|
|
ret = ww_mutex_lock_interruptible(&etnaviv_obj->resv->lock,
|
|
&submit->ticket);
|
|
if (ret == -EALREADY)
|
|
DRM_ERROR("BO at index %u already on submit list\n",
|
|
i);
|
|
if (ret)
|
|
goto fail;
|
|
submit->bos[i].flags |= BO_LOCKED;
|
|
}
|
|
}
|
|
|
|
ww_acquire_done(&submit->ticket);
|
|
|
|
return 0;
|
|
|
|
fail:
|
|
for (; i >= 0; i--)
|
|
submit_unlock_object(submit, i);
|
|
|
|
if (slow_locked > 0)
|
|
submit_unlock_object(submit, slow_locked);
|
|
|
|
if (ret == -EDEADLK) {
|
|
struct etnaviv_gem_object *etnaviv_obj;
|
|
|
|
etnaviv_obj = submit->bos[contended].obj;
|
|
|
|
/* we lost out in a seqno race, lock and retry.. */
|
|
ret = ww_mutex_lock_slow_interruptible(&etnaviv_obj->resv->lock,
|
|
&submit->ticket);
|
|
if (!ret) {
|
|
submit->bos[contended].flags |= BO_LOCKED;
|
|
slow_locked = contended;
|
|
goto retry;
|
|
}
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int submit_fence_sync(const struct etnaviv_gem_submit *submit)
|
|
{
|
|
unsigned int context = submit->gpu->fence_context;
|
|
int i, ret = 0;
|
|
|
|
for (i = 0; i < submit->nr_bos; i++) {
|
|
struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
|
|
bool write = submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE;
|
|
|
|
ret = etnaviv_gpu_fence_sync_obj(etnaviv_obj, context, write);
|
|
if (ret)
|
|
break;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void submit_unpin_objects(struct etnaviv_gem_submit *submit)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < submit->nr_bos; i++) {
|
|
if (submit->bos[i].flags & BO_PINNED)
|
|
etnaviv_gem_mapping_unreference(submit->bos[i].mapping);
|
|
|
|
submit->bos[i].mapping = NULL;
|
|
submit->bos[i].flags &= ~BO_PINNED;
|
|
}
|
|
}
|
|
|
|
static int submit_pin_objects(struct etnaviv_gem_submit *submit)
|
|
{
|
|
int i, ret = 0;
|
|
|
|
for (i = 0; i < submit->nr_bos; i++) {
|
|
struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
|
|
struct etnaviv_vram_mapping *mapping;
|
|
|
|
mapping = etnaviv_gem_mapping_get(&etnaviv_obj->base,
|
|
submit->gpu);
|
|
if (IS_ERR(mapping)) {
|
|
ret = PTR_ERR(mapping);
|
|
break;
|
|
}
|
|
|
|
submit->bos[i].flags |= BO_PINNED;
|
|
submit->bos[i].mapping = mapping;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int submit_bo(struct etnaviv_gem_submit *submit, u32 idx,
|
|
struct etnaviv_gem_submit_bo **bo)
|
|
{
|
|
if (idx >= submit->nr_bos) {
|
|
DRM_ERROR("invalid buffer index: %u (out of %u)\n",
|
|
idx, submit->nr_bos);
|
|
return -EINVAL;
|
|
}
|
|
|
|
*bo = &submit->bos[idx];
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* process the reloc's and patch up the cmdstream as needed: */
|
|
static int submit_reloc(struct etnaviv_gem_submit *submit, void *stream,
|
|
u32 size, const struct drm_etnaviv_gem_submit_reloc *relocs,
|
|
u32 nr_relocs)
|
|
{
|
|
u32 i, last_offset = 0;
|
|
u32 *ptr = stream;
|
|
int ret;
|
|
|
|
for (i = 0; i < nr_relocs; i++) {
|
|
const struct drm_etnaviv_gem_submit_reloc *r = relocs + i;
|
|
struct etnaviv_gem_submit_bo *bo;
|
|
u32 off;
|
|
|
|
if (unlikely(r->flags)) {
|
|
DRM_ERROR("invalid reloc flags\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (r->submit_offset % 4) {
|
|
DRM_ERROR("non-aligned reloc offset: %u\n",
|
|
r->submit_offset);
|
|
return -EINVAL;
|
|
}
|
|
|
|
/* offset in dwords: */
|
|
off = r->submit_offset / 4;
|
|
|
|
if ((off >= size ) ||
|
|
(off < last_offset)) {
|
|
DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
|
|
return -EINVAL;
|
|
}
|
|
|
|
ret = submit_bo(submit, r->reloc_idx, &bo);
|
|
if (ret)
|
|
return ret;
|
|
|
|
if (r->reloc_offset >= bo->obj->base.size - sizeof(*ptr)) {
|
|
DRM_ERROR("relocation %u outside object", i);
|
|
return -EINVAL;
|
|
}
|
|
|
|
ptr[off] = bo->mapping->iova + r->reloc_offset;
|
|
|
|
last_offset = off;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void submit_cleanup(struct etnaviv_gem_submit *submit)
|
|
{
|
|
unsigned i;
|
|
|
|
for (i = 0; i < submit->nr_bos; i++) {
|
|
struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
|
|
|
|
submit_unlock_object(submit, i);
|
|
drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
|
|
}
|
|
|
|
ww_acquire_fini(&submit->ticket);
|
|
kfree(submit);
|
|
}
|
|
|
|
int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
|
|
struct drm_file *file)
|
|
{
|
|
struct etnaviv_drm_private *priv = dev->dev_private;
|
|
struct drm_etnaviv_gem_submit *args = data;
|
|
struct drm_etnaviv_gem_submit_reloc *relocs;
|
|
struct drm_etnaviv_gem_submit_bo *bos;
|
|
struct etnaviv_gem_submit *submit;
|
|
struct etnaviv_cmdbuf *cmdbuf;
|
|
struct etnaviv_gpu *gpu;
|
|
void *stream;
|
|
int ret;
|
|
|
|
if (args->pipe >= ETNA_MAX_PIPES)
|
|
return -EINVAL;
|
|
|
|
gpu = priv->gpu[args->pipe];
|
|
if (!gpu)
|
|
return -ENXIO;
|
|
|
|
if (args->stream_size % 4) {
|
|
DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
|
|
args->stream_size);
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (args->exec_state != ETNA_PIPE_3D &&
|
|
args->exec_state != ETNA_PIPE_2D &&
|
|
args->exec_state != ETNA_PIPE_VG) {
|
|
DRM_ERROR("invalid exec_state: 0x%x\n", args->exec_state);
|
|
return -EINVAL;
|
|
}
|
|
|
|
/*
|
|
* Copy the command submission and bo array to kernel space in
|
|
* one go, and do this outside of any locks.
|
|
*/
|
|
bos = drm_malloc_ab(args->nr_bos, sizeof(*bos));
|
|
relocs = drm_malloc_ab(args->nr_relocs, sizeof(*relocs));
|
|
stream = drm_malloc_ab(1, args->stream_size);
|
|
cmdbuf = etnaviv_cmdbuf_new(gpu->cmdbuf_suballoc,
|
|
ALIGN(args->stream_size, 8) + 8,
|
|
args->nr_bos);
|
|
if (!bos || !relocs || !stream || !cmdbuf) {
|
|
ret = -ENOMEM;
|
|
goto err_submit_cmds;
|
|
}
|
|
|
|
cmdbuf->exec_state = args->exec_state;
|
|
cmdbuf->ctx = file->driver_priv;
|
|
|
|
ret = copy_from_user(bos, u64_to_user_ptr(args->bos),
|
|
args->nr_bos * sizeof(*bos));
|
|
if (ret) {
|
|
ret = -EFAULT;
|
|
goto err_submit_cmds;
|
|
}
|
|
|
|
ret = copy_from_user(relocs, u64_to_user_ptr(args->relocs),
|
|
args->nr_relocs * sizeof(*relocs));
|
|
if (ret) {
|
|
ret = -EFAULT;
|
|
goto err_submit_cmds;
|
|
}
|
|
|
|
ret = copy_from_user(stream, u64_to_user_ptr(args->stream),
|
|
args->stream_size);
|
|
if (ret) {
|
|
ret = -EFAULT;
|
|
goto err_submit_cmds;
|
|
}
|
|
|
|
submit = submit_create(dev, gpu, args->nr_bos);
|
|
if (!submit) {
|
|
ret = -ENOMEM;
|
|
goto err_submit_cmds;
|
|
}
|
|
|
|
ret = submit_lookup_objects(submit, file, bos, args->nr_bos);
|
|
if (ret)
|
|
goto err_submit_objects;
|
|
|
|
ret = submit_lock_objects(submit);
|
|
if (ret)
|
|
goto err_submit_objects;
|
|
|
|
if (!etnaviv_cmd_validate_one(gpu, stream, args->stream_size / 4,
|
|
relocs, args->nr_relocs)) {
|
|
ret = -EINVAL;
|
|
goto err_submit_objects;
|
|
}
|
|
|
|
ret = submit_fence_sync(submit);
|
|
if (ret)
|
|
goto err_submit_objects;
|
|
|
|
ret = submit_pin_objects(submit);
|
|
if (ret)
|
|
goto out;
|
|
|
|
ret = submit_reloc(submit, stream, args->stream_size / 4,
|
|
relocs, args->nr_relocs);
|
|
if (ret)
|
|
goto out;
|
|
|
|
memcpy(cmdbuf->vaddr, stream, args->stream_size);
|
|
cmdbuf->user_size = ALIGN(args->stream_size, 8);
|
|
|
|
ret = etnaviv_gpu_submit(gpu, submit, cmdbuf);
|
|
if (ret == 0)
|
|
cmdbuf = NULL;
|
|
|
|
args->fence = submit->fence;
|
|
|
|
out:
|
|
submit_unpin_objects(submit);
|
|
|
|
/*
|
|
* If we're returning -EAGAIN, it may be due to the userptr code
|
|
* wanting to run its workqueue outside of any locks. Flush our
|
|
* workqueue to ensure that it is run in a timely manner.
|
|
*/
|
|
if (ret == -EAGAIN)
|
|
flush_workqueue(priv->wq);
|
|
|
|
err_submit_objects:
|
|
submit_cleanup(submit);
|
|
|
|
err_submit_cmds:
|
|
/* if we still own the cmdbuf */
|
|
if (cmdbuf)
|
|
etnaviv_cmdbuf_free(cmdbuf);
|
|
if (stream)
|
|
drm_free_large(stream);
|
|
if (bos)
|
|
drm_free_large(bos);
|
|
if (relocs)
|
|
drm_free_large(relocs);
|
|
|
|
return ret;
|
|
}
|