2015-03-18 03:46:04 -06:00
|
|
|
/*
|
|
|
|
* Copyright © 2008-2015 Intel Corporation
|
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
|
|
* to deal in the Software without restriction, including without limitation
|
|
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice (including the next
|
|
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
|
|
* Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
|
|
* IN THE SOFTWARE.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/oom.h>
|
|
|
|
#include <linux/shmem_fs.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/swap.h>
|
|
|
|
#include <linux/pci.h>
|
|
|
|
#include <linux/dma-buf.h>
|
2016-04-04 07:46:43 -06:00
|
|
|
#include <linux/vmalloc.h>
|
2015-03-18 03:46:04 -06:00
|
|
|
#include <drm/drmP.h>
|
|
|
|
#include <drm/i915_drm.h>
|
|
|
|
|
|
|
|
#include "i915_drv.h"
|
|
|
|
#include "i915_trace.h"
|
|
|
|
|
2017-04-07 04:49:35 -06:00
|
|
|
static bool shrinker_lock(struct drm_i915_private *dev_priv, bool *unlock)
|
2015-03-18 03:46:04 -06:00
|
|
|
{
|
2017-04-07 04:49:35 -06:00
|
|
|
switch (mutex_trylock_recursive(&dev_priv->drm.struct_mutex)) {
|
2016-12-13 10:35:09 -07:00
|
|
|
case MUTEX_TRYLOCK_RECURSIVE:
|
2016-10-28 06:58:37 -06:00
|
|
|
*unlock = false;
|
2016-12-13 10:35:09 -07:00
|
|
|
return true;
|
2017-06-09 05:03:49 -06:00
|
|
|
|
|
|
|
case MUTEX_TRYLOCK_FAILED:
|
2017-08-04 04:41:35 -06:00
|
|
|
*unlock = false;
|
|
|
|
preempt_disable();
|
2017-06-09 05:03:49 -06:00
|
|
|
do {
|
|
|
|
cpu_relax();
|
|
|
|
if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
|
|
|
|
*unlock = true;
|
2017-08-04 04:41:35 -06:00
|
|
|
break;
|
2017-06-09 05:03:49 -06:00
|
|
|
}
|
|
|
|
} while (!need_resched());
|
2017-08-04 04:41:35 -06:00
|
|
|
preempt_enable();
|
|
|
|
return *unlock;
|
2017-06-09 05:03:49 -06:00
|
|
|
|
2017-08-04 04:41:35 -06:00
|
|
|
case MUTEX_TRYLOCK_SUCCESS:
|
|
|
|
*unlock = true;
|
|
|
|
return true;
|
2016-10-28 06:58:37 -06:00
|
|
|
}
|
|
|
|
|
2016-12-13 10:35:09 -07:00
|
|
|
BUG();
|
2016-10-28 06:58:37 -06:00
|
|
|
}
|
|
|
|
|
2017-04-07 04:49:35 -06:00
|
|
|
static void shrinker_unlock(struct drm_i915_private *dev_priv, bool unlock)
|
2017-04-07 04:49:34 -06:00
|
|
|
{
|
|
|
|
if (!unlock)
|
|
|
|
return;
|
|
|
|
|
2017-04-07 04:49:35 -06:00
|
|
|
mutex_unlock(&dev_priv->drm.struct_mutex);
|
2017-04-07 04:49:34 -06:00
|
|
|
}
|
|
|
|
|
2015-12-04 08:58:54 -07:00
|
|
|
static bool swap_available(void)
|
|
|
|
{
|
|
|
|
return get_nr_swap_pages() > 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool can_release_pages(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
2016-11-01 08:44:10 -06:00
|
|
|
/* Consider only shrinkable ojects. */
|
|
|
|
if (!i915_gem_object_is_shrinkable(obj))
|
2016-04-20 05:09:52 -06:00
|
|
|
return false;
|
|
|
|
|
2015-12-04 08:58:54 -07:00
|
|
|
/* Only report true if by unbinding the object and putting its pages
|
|
|
|
* we can actually make forward progress towards freeing physical
|
|
|
|
* pages.
|
|
|
|
*
|
|
|
|
* If the pages are pinned for any other reason than being bound
|
|
|
|
* to the GPU, simply unbinding from the GPU is not going to succeed
|
|
|
|
* in releasing our pin count on the pages themselves.
|
|
|
|
*/
|
2016-10-28 06:58:37 -06:00
|
|
|
if (atomic_read(&obj->mm.pages_pin_count) > obj->bind_count)
|
2016-08-04 00:52:26 -06:00
|
|
|
return false;
|
|
|
|
|
2017-10-13 14:26:16 -06:00
|
|
|
/* If any vma are "permanently" pinned, it will prevent us from
|
|
|
|
* reclaiming the obj->mm.pages. We only allow scanout objects to claim
|
|
|
|
* a permanent pin, along with a few others like the context objects.
|
|
|
|
* To simplify the scan, and to avoid walking the list of vma under the
|
|
|
|
* object, we just check the count of its permanently pinned.
|
|
|
|
*/
|
2017-10-16 05:40:37 -06:00
|
|
|
if (READ_ONCE(obj->pin_global))
|
2015-12-04 08:58:54 -07:00
|
|
|
return false;
|
|
|
|
|
|
|
|
/* We can only return physical pages to the system if we can either
|
|
|
|
* discard the contents (because the user has marked them as being
|
|
|
|
* purgeable) or if we can move their contents out to swap.
|
|
|
|
*/
|
2016-10-28 06:58:35 -06:00
|
|
|
return swap_available() || obj->mm.madv == I915_MADV_DONTNEED;
|
2015-12-04 08:58:54 -07:00
|
|
|
}
|
|
|
|
|
2016-10-28 06:58:36 -06:00
|
|
|
static bool unsafe_drop_pages(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
|
|
|
if (i915_gem_object_unbind(obj) == 0)
|
2016-11-01 06:11:34 -06:00
|
|
|
__i915_gem_object_put_pages(obj, I915_MM_SHRINKER);
|
2017-10-13 14:26:13 -06:00
|
|
|
return !i915_gem_object_has_pages(obj);
|
2015-12-04 08:58:54 -07:00
|
|
|
}
|
|
|
|
|
2015-03-18 07:47:59 -06:00
|
|
|
/**
|
|
|
|
* i915_gem_shrink - Shrink buffer object caches
|
|
|
|
* @dev_priv: i915 device
|
|
|
|
* @target: amount of memory to make available, in pages
|
2017-09-06 17:19:30 -06:00
|
|
|
* @nr_scanned: optional output for number of pages scanned (incremental)
|
2015-03-18 07:47:59 -06:00
|
|
|
* @flags: control flags for selecting cache types
|
|
|
|
*
|
|
|
|
* This function is the main interface to the shrinker. It will try to release
|
|
|
|
* up to @target pages of main memory backing storage from buffer objects.
|
|
|
|
* Selection of the specific caches can be done with @flags. This is e.g. useful
|
|
|
|
* when purgeable objects should be removed from caches preferentially.
|
|
|
|
*
|
|
|
|
* Note that it's not guaranteed that released amount is actually available as
|
|
|
|
* free system memory - the pages might still be in-used to due to other reasons
|
|
|
|
* (like cpu mmaps) or the mm core has reused them before we could grab them.
|
|
|
|
* Therefore code that needs to explicitly shrink buffer objects caches (e.g. to
|
|
|
|
* avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all().
|
|
|
|
*
|
|
|
|
* Also note that any kind of pinning (both per-vma address space pins and
|
|
|
|
* backing storage pins at the buffer object level) result in the shrinker code
|
|
|
|
* having to skip the object.
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* The number of pages of backing storage actually released.
|
|
|
|
*/
|
2015-03-18 03:46:04 -06:00
|
|
|
unsigned long
|
|
|
|
i915_gem_shrink(struct drm_i915_private *dev_priv,
|
2017-09-06 17:19:30 -06:00
|
|
|
unsigned long target,
|
|
|
|
unsigned long *nr_scanned,
|
|
|
|
unsigned flags)
|
2015-03-18 03:46:04 -06:00
|
|
|
{
|
|
|
|
const struct {
|
|
|
|
struct list_head *list;
|
|
|
|
unsigned int bit;
|
|
|
|
} phases[] = {
|
|
|
|
{ &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
|
|
|
|
{ &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
|
|
|
|
{ NULL, 0 },
|
|
|
|
}, *phase;
|
|
|
|
unsigned long count = 0;
|
2017-09-06 17:19:30 -06:00
|
|
|
unsigned long scanned = 0;
|
2016-10-28 06:58:37 -06:00
|
|
|
bool unlock;
|
|
|
|
|
2017-04-07 04:49:35 -06:00
|
|
|
if (!shrinker_lock(dev_priv, &unlock))
|
2016-10-28 06:58:37 -06:00
|
|
|
return 0;
|
2015-03-18 03:46:04 -06:00
|
|
|
|
2017-11-08 02:44:00 -07:00
|
|
|
/*
|
|
|
|
* When shrinking the active list, also consider active contexts.
|
|
|
|
* Active contexts are pinned until they are retired, and so can
|
|
|
|
* not be simply unbound to retire and unpin their pages. To shrink
|
|
|
|
* the contexts, we must wait until the gpu is idle.
|
|
|
|
*
|
|
|
|
* We don't care about errors here; if we cannot wait upon the GPU,
|
|
|
|
* we will free as much as we can and hope to get a second chance.
|
|
|
|
*/
|
|
|
|
if (flags & I915_SHRINK_ACTIVE)
|
|
|
|
i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED);
|
|
|
|
|
2015-10-01 05:18:26 -06:00
|
|
|
trace_i915_gem_shrink(dev_priv, target, flags);
|
2016-05-06 08:40:21 -06:00
|
|
|
i915_gem_retire_requests(dev_priv);
|
2015-10-01 05:18:26 -06:00
|
|
|
|
2016-05-02 02:40:28 -06:00
|
|
|
/*
|
|
|
|
* Unbinding of objects will require HW access; Let us not wake the
|
|
|
|
* device just to recover a little memory. If absolutely necessary,
|
|
|
|
* we will force the wake during oom-notifier.
|
|
|
|
*/
|
|
|
|
if ((flags & I915_SHRINK_BOUND) &&
|
|
|
|
!intel_runtime_pm_get_if_in_use(dev_priv))
|
|
|
|
flags &= ~I915_SHRINK_BOUND;
|
|
|
|
|
2015-03-18 03:46:04 -06:00
|
|
|
/*
|
|
|
|
* As we may completely rewrite the (un)bound list whilst unbinding
|
|
|
|
* (due to retiring requests) we have to strictly process only
|
|
|
|
* one element of the list at the time, and recheck the list
|
|
|
|
* on every iteration.
|
|
|
|
*
|
|
|
|
* In particular, we must hold a reference whilst removing the
|
|
|
|
* object as we may end up waiting for and/or retiring the objects.
|
|
|
|
* This might release the final reference (held by the active list)
|
|
|
|
* and result in the object being freed from under us. This is
|
|
|
|
* similar to the precautions the eviction code must take whilst
|
|
|
|
* removing objects.
|
|
|
|
*
|
|
|
|
* Also note that although these lists do not hold a reference to
|
|
|
|
* the object we can safely grab one here: The final object
|
|
|
|
* unreferencing and the bound_list are both protected by the
|
|
|
|
* dev->struct_mutex and so we won't ever be able to observe an
|
|
|
|
* object on the bound_list with a reference count equals 0.
|
|
|
|
*/
|
|
|
|
for (phase = phases; phase->list; phase++) {
|
|
|
|
struct list_head still_in_list;
|
2016-07-26 05:01:51 -06:00
|
|
|
struct drm_i915_gem_object *obj;
|
2015-03-18 03:46:04 -06:00
|
|
|
|
|
|
|
if ((flags & phase->bit) == 0)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
INIT_LIST_HEAD(&still_in_list);
|
2017-10-16 05:40:37 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We serialize our access to unreferenced objects through
|
|
|
|
* the use of the struct_mutex. While the objects are not
|
|
|
|
* yet freed (due to RCU then a workqueue) we still want
|
|
|
|
* to be able to shrink their pages, so they remain on
|
|
|
|
* the unbound/bound list until actually freed.
|
|
|
|
*/
|
|
|
|
spin_lock(&dev_priv->mm.obj_lock);
|
2016-07-26 05:01:51 -06:00
|
|
|
while (count < target &&
|
|
|
|
(obj = list_first_entry_or_null(phase->list,
|
|
|
|
typeof(*obj),
|
2017-10-16 05:40:37 -06:00
|
|
|
mm.link))) {
|
|
|
|
list_move_tail(&obj->mm.link, &still_in_list);
|
2015-03-18 03:46:04 -06:00
|
|
|
|
|
|
|
if (flags & I915_SHRINK_PURGEABLE &&
|
2016-10-28 06:58:35 -06:00
|
|
|
obj->mm.madv != I915_MADV_DONTNEED)
|
2015-03-18 03:46:04 -06:00
|
|
|
continue;
|
|
|
|
|
2016-04-08 05:11:12 -06:00
|
|
|
if (flags & I915_SHRINK_VMAPS &&
|
2016-10-28 06:58:35 -06:00
|
|
|
!is_vmalloc_addr(obj->mm.mapping))
|
2016-04-08 05:11:12 -06:00
|
|
|
continue;
|
|
|
|
|
2016-10-12 06:48:24 -06:00
|
|
|
if (!(flags & I915_SHRINK_ACTIVE) &&
|
|
|
|
(i915_gem_object_is_active(obj) ||
|
2017-03-01 08:41:28 -07:00
|
|
|
i915_gem_object_is_framebuffer(obj)))
|
2015-10-01 05:18:29 -06:00
|
|
|
continue;
|
|
|
|
|
2015-12-04 08:58:54 -07:00
|
|
|
if (!can_release_pages(obj))
|
|
|
|
continue;
|
|
|
|
|
2017-10-16 05:40:37 -06:00
|
|
|
spin_unlock(&dev_priv->mm.obj_lock);
|
|
|
|
|
2016-10-28 06:58:37 -06:00
|
|
|
if (unsafe_drop_pages(obj)) {
|
2016-10-31 06:40:48 -06:00
|
|
|
/* May arrive from get_pages on another bo */
|
|
|
|
mutex_lock_nested(&obj->mm.lock,
|
2016-11-01 06:11:34 -06:00
|
|
|
I915_MM_SHRINKER);
|
2017-10-13 14:26:13 -06:00
|
|
|
if (!i915_gem_object_has_pages(obj)) {
|
2016-10-28 06:58:37 -06:00
|
|
|
__i915_gem_object_invalidate(obj);
|
|
|
|
count += obj->base.size >> PAGE_SHIFT;
|
|
|
|
}
|
|
|
|
mutex_unlock(&obj->mm.lock);
|
|
|
|
}
|
2017-10-13 14:26:18 -06:00
|
|
|
scanned += obj->base.size >> PAGE_SHIFT;
|
2017-10-16 05:40:37 -06:00
|
|
|
|
|
|
|
spin_lock(&dev_priv->mm.obj_lock);
|
2015-03-18 03:46:04 -06:00
|
|
|
}
|
2016-11-01 02:48:43 -06:00
|
|
|
list_splice_tail(&still_in_list, phase->list);
|
2017-10-16 05:40:37 -06:00
|
|
|
spin_unlock(&dev_priv->mm.obj_lock);
|
2015-03-18 03:46:04 -06:00
|
|
|
}
|
|
|
|
|
2016-05-02 02:40:28 -06:00
|
|
|
if (flags & I915_SHRINK_BOUND)
|
|
|
|
intel_runtime_pm_put(dev_priv);
|
|
|
|
|
2016-05-06 08:40:21 -06:00
|
|
|
i915_gem_retire_requests(dev_priv);
|
2016-10-28 06:58:37 -06:00
|
|
|
|
2017-04-07 04:49:35 -06:00
|
|
|
shrinker_unlock(dev_priv, unlock);
|
2015-10-01 05:18:27 -06:00
|
|
|
|
2017-09-06 17:19:30 -06:00
|
|
|
if (nr_scanned)
|
|
|
|
*nr_scanned += scanned;
|
2015-03-18 03:46:04 -06:00
|
|
|
return count;
|
|
|
|
}
|
|
|
|
|
2015-03-18 07:47:59 -06:00
|
|
|
/**
|
2015-10-06 06:47:55 -06:00
|
|
|
* i915_gem_shrink_all - Shrink buffer object caches completely
|
2015-03-18 07:47:59 -06:00
|
|
|
* @dev_priv: i915 device
|
|
|
|
*
|
|
|
|
* This is a simple wraper around i915_gem_shrink() to aggressively shrink all
|
|
|
|
* caches completely. It also first waits for and retires all outstanding
|
|
|
|
* requests to also be able to release backing storage for active objects.
|
|
|
|
*
|
|
|
|
* This should only be used in code to intentionally quiescent the gpu or as a
|
|
|
|
* last-ditch effort when memory seems to have run out.
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* The number of pages of backing storage actually released.
|
|
|
|
*/
|
2015-03-18 03:46:04 -06:00
|
|
|
unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
|
|
|
|
{
|
drm/i915: Enable lockless lookup of request tracking via RCU
If we enable RCU for the requests (providing a grace period where we can
inspect a "dead" request before it is freed), we can allow callers to
carefully perform lockless lookup of an active request.
However, by enabling deferred freeing of requests, we can potentially
hog a lot of memory when dealing with tens of thousands of requests per
second - with a quick insertion of a synchronize_rcu() inside our
shrinker callback, that issue disappears.
v2: Currently, it is our responsibility to handle reclaim i.e. to avoid
hogging memory with the delayed slab frees. At the moment, we wait for a
grace period in the shrinker, and block for all RCU callbacks on oom.
Suggested alternatives focus on flushing our RCU callback when we have a
certain number of outstanding request frees, and blocking on that flush
after a second high watermark. (So rather than wait for the system to
run out of memory, we stop issuing requests - both are nondeterministic.)
Paul E. McKenney wrote:
Another approach is synchronize_rcu() after some largish number of
requests. The advantage of this approach is that it throttles the
production of callbacks at the source. The corresponding disadvantage
is that it slows things up.
Another approach is to use call_rcu(), but if the previous call_rcu()
is still in flight, block waiting for it. Yet another approach is
the get_state_synchronize_rcu() / cond_synchronize_rcu() pair. The
idea is to do something like this:
cond_synchronize_rcu(cookie);
cookie = get_state_synchronize_rcu();
You would of course do an initial get_state_synchronize_rcu() to
get things going. This would not block unless there was less than
one grace period's worth of time between invocations. But this
assumes a busy system, where there is almost always a grace period
in flight. But you can make that happen as follows:
cond_synchronize_rcu(cookie);
cookie = get_state_synchronize_rcu();
call_rcu(&my_rcu_head, noop_function);
Note that you need additional code to make sure that the old callback
has completed before doing a new one. Setting and clearing a flag
with appropriate memory ordering control suffices (e.g,. smp_load_acquire()
and smp_store_release()).
v3: More comments on compiler and processor order of operations within
the RCU lookup and discover we can use rcu_access_pointer() here instead.
v4: Wrap i915_gem_active_get_rcu() to take the rcu_read_lock itself.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Maarten Lankhorst <maarten.lankhorst@linux.intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Link: http://patchwork.freedesktop.org/patch/msgid/1470324762-2545-25-git-send-email-chris@chris-wilson.co.uk
2016-08-04 09:32:41 -06:00
|
|
|
unsigned long freed;
|
|
|
|
|
2017-02-08 03:47:10 -07:00
|
|
|
intel_runtime_pm_get(dev_priv);
|
2017-09-06 17:19:30 -06:00
|
|
|
freed = i915_gem_shrink(dev_priv, -1UL, NULL,
|
drm/i915: Enable lockless lookup of request tracking via RCU
If we enable RCU for the requests (providing a grace period where we can
inspect a "dead" request before it is freed), we can allow callers to
carefully perform lockless lookup of an active request.
However, by enabling deferred freeing of requests, we can potentially
hog a lot of memory when dealing with tens of thousands of requests per
second - with a quick insertion of a synchronize_rcu() inside our
shrinker callback, that issue disappears.
v2: Currently, it is our responsibility to handle reclaim i.e. to avoid
hogging memory with the delayed slab frees. At the moment, we wait for a
grace period in the shrinker, and block for all RCU callbacks on oom.
Suggested alternatives focus on flushing our RCU callback when we have a
certain number of outstanding request frees, and blocking on that flush
after a second high watermark. (So rather than wait for the system to
run out of memory, we stop issuing requests - both are nondeterministic.)
Paul E. McKenney wrote:
Another approach is synchronize_rcu() after some largish number of
requests. The advantage of this approach is that it throttles the
production of callbacks at the source. The corresponding disadvantage
is that it slows things up.
Another approach is to use call_rcu(), but if the previous call_rcu()
is still in flight, block waiting for it. Yet another approach is
the get_state_synchronize_rcu() / cond_synchronize_rcu() pair. The
idea is to do something like this:
cond_synchronize_rcu(cookie);
cookie = get_state_synchronize_rcu();
You would of course do an initial get_state_synchronize_rcu() to
get things going. This would not block unless there was less than
one grace period's worth of time between invocations. But this
assumes a busy system, where there is almost always a grace period
in flight. But you can make that happen as follows:
cond_synchronize_rcu(cookie);
cookie = get_state_synchronize_rcu();
call_rcu(&my_rcu_head, noop_function);
Note that you need additional code to make sure that the old callback
has completed before doing a new one. Setting and clearing a flag
with appropriate memory ordering control suffices (e.g,. smp_load_acquire()
and smp_store_release()).
v3: More comments on compiler and processor order of operations within
the RCU lookup and discover we can use rcu_access_pointer() here instead.
v4: Wrap i915_gem_active_get_rcu() to take the rcu_read_lock itself.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Maarten Lankhorst <maarten.lankhorst@linux.intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Link: http://patchwork.freedesktop.org/patch/msgid/1470324762-2545-25-git-send-email-chris@chris-wilson.co.uk
2016-08-04 09:32:41 -06:00
|
|
|
I915_SHRINK_BOUND |
|
|
|
|
I915_SHRINK_UNBOUND |
|
|
|
|
I915_SHRINK_ACTIVE);
|
2017-02-08 03:47:10 -07:00
|
|
|
intel_runtime_pm_put(dev_priv);
|
|
|
|
|
drm/i915: Enable lockless lookup of request tracking via RCU
If we enable RCU for the requests (providing a grace period where we can
inspect a "dead" request before it is freed), we can allow callers to
carefully perform lockless lookup of an active request.
However, by enabling deferred freeing of requests, we can potentially
hog a lot of memory when dealing with tens of thousands of requests per
second - with a quick insertion of a synchronize_rcu() inside our
shrinker callback, that issue disappears.
v2: Currently, it is our responsibility to handle reclaim i.e. to avoid
hogging memory with the delayed slab frees. At the moment, we wait for a
grace period in the shrinker, and block for all RCU callbacks on oom.
Suggested alternatives focus on flushing our RCU callback when we have a
certain number of outstanding request frees, and blocking on that flush
after a second high watermark. (So rather than wait for the system to
run out of memory, we stop issuing requests - both are nondeterministic.)
Paul E. McKenney wrote:
Another approach is synchronize_rcu() after some largish number of
requests. The advantage of this approach is that it throttles the
production of callbacks at the source. The corresponding disadvantage
is that it slows things up.
Another approach is to use call_rcu(), but if the previous call_rcu()
is still in flight, block waiting for it. Yet another approach is
the get_state_synchronize_rcu() / cond_synchronize_rcu() pair. The
idea is to do something like this:
cond_synchronize_rcu(cookie);
cookie = get_state_synchronize_rcu();
You would of course do an initial get_state_synchronize_rcu() to
get things going. This would not block unless there was less than
one grace period's worth of time between invocations. But this
assumes a busy system, where there is almost always a grace period
in flight. But you can make that happen as follows:
cond_synchronize_rcu(cookie);
cookie = get_state_synchronize_rcu();
call_rcu(&my_rcu_head, noop_function);
Note that you need additional code to make sure that the old callback
has completed before doing a new one. Setting and clearing a flag
with appropriate memory ordering control suffices (e.g,. smp_load_acquire()
and smp_store_release()).
v3: More comments on compiler and processor order of operations within
the RCU lookup and discover we can use rcu_access_pointer() here instead.
v4: Wrap i915_gem_active_get_rcu() to take the rcu_read_lock itself.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Maarten Lankhorst <maarten.lankhorst@linux.intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Link: http://patchwork.freedesktop.org/patch/msgid/1470324762-2545-25-git-send-email-chris@chris-wilson.co.uk
2016-08-04 09:32:41 -06:00
|
|
|
return freed;
|
2015-03-18 03:46:04 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned long
|
|
|
|
i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
|
|
|
|
{
|
2017-10-13 14:26:19 -06:00
|
|
|
struct drm_i915_private *i915 =
|
2015-03-18 03:46:04 -06:00
|
|
|
container_of(shrinker, struct drm_i915_private, mm.shrinker);
|
|
|
|
struct drm_i915_gem_object *obj;
|
2017-10-13 14:26:19 -06:00
|
|
|
unsigned long num_objects = 0;
|
2017-10-16 05:40:37 -06:00
|
|
|
unsigned long count = 0;
|
2016-07-01 10:23:10 -06:00
|
|
|
|
2017-10-13 14:26:19 -06:00
|
|
|
spin_lock(&i915->mm.obj_lock);
|
|
|
|
list_for_each_entry(obj, &i915->mm.unbound_list, mm.link)
|
|
|
|
if (can_release_pages(obj)) {
|
2015-03-18 03:46:04 -06:00
|
|
|
count += obj->base.size >> PAGE_SHIFT;
|
2017-10-13 14:26:19 -06:00
|
|
|
num_objects++;
|
|
|
|
}
|
2015-03-18 03:46:04 -06:00
|
|
|
|
2017-10-13 14:26:19 -06:00
|
|
|
list_for_each_entry(obj, &i915->mm.bound_list, mm.link)
|
|
|
|
if (!i915_gem_object_is_active(obj) && can_release_pages(obj)) {
|
2015-03-18 03:46:04 -06:00
|
|
|
count += obj->base.size >> PAGE_SHIFT;
|
2017-10-13 14:26:19 -06:00
|
|
|
num_objects++;
|
|
|
|
}
|
|
|
|
spin_unlock(&i915->mm.obj_lock);
|
|
|
|
|
|
|
|
/* Update our preferred vmscan batch size for the next pass.
|
|
|
|
* Our rough guess for an effective batch size is roughly 2
|
|
|
|
* available GEM objects worth of pages. That is we don't want
|
|
|
|
* the shrinker to fire, until it is worth the cost of freeing an
|
|
|
|
* entire GEM object.
|
|
|
|
*/
|
|
|
|
if (num_objects) {
|
|
|
|
unsigned long avg = 2 * count / num_objects;
|
|
|
|
|
|
|
|
i915->mm.shrinker.batch =
|
|
|
|
max((i915->mm.shrinker.batch + avg) >> 1,
|
|
|
|
128ul /* default SHRINK_BATCH */);
|
|
|
|
}
|
2015-03-18 03:46:04 -06:00
|
|
|
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned long
|
|
|
|
i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv =
|
|
|
|
container_of(shrinker, struct drm_i915_private, mm.shrinker);
|
|
|
|
unsigned long freed;
|
|
|
|
bool unlock;
|
|
|
|
|
2017-09-06 17:19:30 -06:00
|
|
|
sc->nr_scanned = 0;
|
|
|
|
|
2017-04-07 04:49:35 -06:00
|
|
|
if (!shrinker_lock(dev_priv, &unlock))
|
2015-03-18 03:46:04 -06:00
|
|
|
return SHRINK_STOP;
|
|
|
|
|
|
|
|
freed = i915_gem_shrink(dev_priv,
|
|
|
|
sc->nr_to_scan,
|
2017-09-06 17:19:30 -06:00
|
|
|
&sc->nr_scanned,
|
2015-03-18 03:46:04 -06:00
|
|
|
I915_SHRINK_BOUND |
|
|
|
|
I915_SHRINK_UNBOUND |
|
|
|
|
I915_SHRINK_PURGEABLE);
|
|
|
|
if (freed < sc->nr_to_scan)
|
|
|
|
freed += i915_gem_shrink(dev_priv,
|
2017-09-06 17:19:30 -06:00
|
|
|
sc->nr_to_scan - sc->nr_scanned,
|
|
|
|
&sc->nr_scanned,
|
2015-03-18 03:46:04 -06:00
|
|
|
I915_SHRINK_BOUND |
|
|
|
|
I915_SHRINK_UNBOUND);
|
2017-06-01 07:33:29 -06:00
|
|
|
if (freed < sc->nr_to_scan && current_is_kswapd()) {
|
|
|
|
intel_runtime_pm_get(dev_priv);
|
|
|
|
freed += i915_gem_shrink(dev_priv,
|
2017-09-06 17:19:30 -06:00
|
|
|
sc->nr_to_scan - sc->nr_scanned,
|
|
|
|
&sc->nr_scanned,
|
2017-06-01 07:33:29 -06:00
|
|
|
I915_SHRINK_ACTIVE |
|
|
|
|
I915_SHRINK_BOUND |
|
|
|
|
I915_SHRINK_UNBOUND);
|
|
|
|
intel_runtime_pm_put(dev_priv);
|
|
|
|
}
|
2017-04-07 04:49:34 -06:00
|
|
|
|
2017-04-07 04:49:35 -06:00
|
|
|
shrinker_unlock(dev_priv, unlock);
|
2015-03-18 03:46:04 -06:00
|
|
|
|
2017-09-06 17:19:30 -06:00
|
|
|
return sc->nr_scanned ? freed : SHRINK_STOP;
|
2015-03-18 03:46:04 -06:00
|
|
|
}
|
|
|
|
|
2016-04-05 03:22:25 -06:00
|
|
|
static bool
|
2017-04-07 04:49:35 -06:00
|
|
|
shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv, bool *unlock,
|
|
|
|
int timeout_ms)
|
2016-04-05 03:22:25 -06:00
|
|
|
{
|
2016-08-05 03:14:13 -06:00
|
|
|
unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms);
|
|
|
|
|
|
|
|
do {
|
2016-09-09 07:11:49 -06:00
|
|
|
if (i915_gem_wait_for_idle(dev_priv, 0) == 0 &&
|
2017-04-07 04:49:35 -06:00
|
|
|
shrinker_lock(dev_priv, unlock))
|
2016-08-05 03:14:13 -06:00
|
|
|
break;
|
2016-04-05 03:22:25 -06:00
|
|
|
|
|
|
|
schedule_timeout_killable(1);
|
|
|
|
if (fatal_signal_pending(current))
|
|
|
|
return false;
|
2016-08-05 03:14:13 -06:00
|
|
|
|
|
|
|
if (time_after(jiffies, timeout)) {
|
2016-04-05 03:22:25 -06:00
|
|
|
pr_err("Unable to lock GPU to purge memory.\n");
|
|
|
|
return false;
|
|
|
|
}
|
2016-08-05 03:14:13 -06:00
|
|
|
} while (1);
|
2016-04-05 03:22:25 -06:00
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2015-03-18 03:46:04 -06:00
|
|
|
static int
|
|
|
|
i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv =
|
|
|
|
container_of(nb, struct drm_i915_private, mm.oom_notifier);
|
|
|
|
struct drm_i915_gem_object *obj;
|
2016-04-20 05:09:51 -06:00
|
|
|
unsigned long unevictable, bound, unbound, freed_pages;
|
2015-03-18 03:46:04 -06:00
|
|
|
|
|
|
|
freed_pages = i915_gem_shrink_all(dev_priv);
|
|
|
|
|
|
|
|
/* Because we may be allocating inside our own driver, we cannot
|
|
|
|
* assert that there are no objects with pinned pages that are not
|
|
|
|
* being pointed to by hardware.
|
|
|
|
*/
|
2016-04-20 05:09:51 -06:00
|
|
|
unbound = bound = unevictable = 0;
|
2017-10-16 05:40:37 -06:00
|
|
|
spin_lock(&dev_priv->mm.obj_lock);
|
|
|
|
list_for_each_entry(obj, &dev_priv->mm.unbound_list, mm.link) {
|
2016-04-20 05:09:51 -06:00
|
|
|
if (!can_release_pages(obj))
|
|
|
|
unevictable += obj->base.size >> PAGE_SHIFT;
|
2015-03-18 03:46:04 -06:00
|
|
|
else
|
2016-04-20 05:09:51 -06:00
|
|
|
unbound += obj->base.size >> PAGE_SHIFT;
|
2015-03-18 03:46:04 -06:00
|
|
|
}
|
2017-10-16 05:40:37 -06:00
|
|
|
list_for_each_entry(obj, &dev_priv->mm.bound_list, mm.link) {
|
2016-04-20 05:09:51 -06:00
|
|
|
if (!can_release_pages(obj))
|
|
|
|
unevictable += obj->base.size >> PAGE_SHIFT;
|
2015-03-18 03:46:04 -06:00
|
|
|
else
|
2016-04-20 05:09:51 -06:00
|
|
|
bound += obj->base.size >> PAGE_SHIFT;
|
2015-03-18 03:46:04 -06:00
|
|
|
}
|
2017-10-16 05:40:37 -06:00
|
|
|
spin_unlock(&dev_priv->mm.obj_lock);
|
2015-03-18 03:46:04 -06:00
|
|
|
|
|
|
|
if (freed_pages || unbound || bound)
|
2016-04-20 05:09:51 -06:00
|
|
|
pr_info("Purging GPU memory, %lu pages freed, "
|
|
|
|
"%lu pages still pinned.\n",
|
|
|
|
freed_pages, unevictable);
|
2015-03-18 03:46:04 -06:00
|
|
|
if (unbound || bound)
|
2016-04-20 05:09:51 -06:00
|
|
|
pr_err("%lu and %lu pages still available in the "
|
2015-03-18 03:46:04 -06:00
|
|
|
"bound and unbound GPU page lists.\n",
|
|
|
|
bound, unbound);
|
|
|
|
|
|
|
|
*(unsigned long *)ptr += freed_pages;
|
|
|
|
return NOTIFY_DONE;
|
|
|
|
}
|
|
|
|
|
2016-04-04 07:46:43 -06:00
|
|
|
static int
|
|
|
|
i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv =
|
|
|
|
container_of(nb, struct drm_i915_private, mm.vmap_notifier);
|
2016-04-28 02:56:39 -06:00
|
|
|
struct i915_vma *vma, *next;
|
|
|
|
unsigned long freed_pages = 0;
|
2017-04-07 04:49:35 -06:00
|
|
|
bool unlock;
|
2016-04-28 02:56:39 -06:00
|
|
|
int ret;
|
2016-04-04 07:46:43 -06:00
|
|
|
|
2017-04-07 04:49:35 -06:00
|
|
|
if (!shrinker_lock_uninterruptible(dev_priv, &unlock, 5000))
|
2016-04-04 07:46:43 -06:00
|
|
|
return NOTIFY_DONE;
|
|
|
|
|
2016-04-28 02:56:39 -06:00
|
|
|
/* Force everything onto the inactive lists */
|
2016-09-09 07:11:50 -06:00
|
|
|
ret = i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED);
|
2016-04-28 02:56:39 -06:00
|
|
|
if (ret)
|
|
|
|
goto out;
|
|
|
|
|
2016-05-02 02:40:29 -06:00
|
|
|
intel_runtime_pm_get(dev_priv);
|
2017-09-06 17:19:30 -06:00
|
|
|
freed_pages += i915_gem_shrink(dev_priv, -1UL, NULL,
|
2016-04-28 02:56:39 -06:00
|
|
|
I915_SHRINK_BOUND |
|
|
|
|
I915_SHRINK_UNBOUND |
|
|
|
|
I915_SHRINK_ACTIVE |
|
|
|
|
I915_SHRINK_VMAPS);
|
2016-05-02 02:40:29 -06:00
|
|
|
intel_runtime_pm_put(dev_priv);
|
2016-04-28 02:56:39 -06:00
|
|
|
|
|
|
|
/* We also want to clear any cached iomaps as they wrap vmap */
|
|
|
|
list_for_each_entry_safe(vma, next,
|
|
|
|
&dev_priv->ggtt.base.inactive_list, vm_link) {
|
|
|
|
unsigned long count = vma->node.size >> PAGE_SHIFT;
|
|
|
|
if (vma->iomap && i915_vma_unbind(vma) == 0)
|
|
|
|
freed_pages += count;
|
|
|
|
}
|
2016-04-04 07:46:43 -06:00
|
|
|
|
2016-04-28 02:56:39 -06:00
|
|
|
out:
|
2017-04-07 04:49:35 -06:00
|
|
|
shrinker_unlock(dev_priv, unlock);
|
2016-04-04 07:46:43 -06:00
|
|
|
|
|
|
|
*(unsigned long *)ptr += freed_pages;
|
|
|
|
return NOTIFY_DONE;
|
|
|
|
}
|
|
|
|
|
2015-03-18 07:47:59 -06:00
|
|
|
/**
|
|
|
|
* i915_gem_shrinker_init - Initialize i915 shrinker
|
|
|
|
* @dev_priv: i915 device
|
|
|
|
*
|
|
|
|
* This function registers and sets up the i915 shrinker and OOM handler.
|
|
|
|
*/
|
2015-03-18 03:46:04 -06:00
|
|
|
void i915_gem_shrinker_init(struct drm_i915_private *dev_priv)
|
|
|
|
{
|
|
|
|
dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
|
|
|
|
dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count;
|
|
|
|
dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS;
|
2017-10-13 14:26:19 -06:00
|
|
|
dev_priv->mm.shrinker.batch = 4096;
|
2016-01-19 06:26:28 -07:00
|
|
|
WARN_ON(register_shrinker(&dev_priv->mm.shrinker));
|
2015-03-18 03:46:04 -06:00
|
|
|
|
|
|
|
dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
|
2016-01-19 06:26:28 -07:00
|
|
|
WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier));
|
2016-04-04 07:46:43 -06:00
|
|
|
|
|
|
|
dev_priv->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap;
|
|
|
|
WARN_ON(register_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
|
2016-01-19 06:26:28 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* i915_gem_shrinker_cleanup - Clean up i915 shrinker
|
|
|
|
* @dev_priv: i915 device
|
|
|
|
*
|
|
|
|
* This function unregisters the i915 shrinker and OOM handler.
|
|
|
|
*/
|
|
|
|
void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv)
|
|
|
|
{
|
2016-04-04 07:46:43 -06:00
|
|
|
WARN_ON(unregister_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
|
2016-01-19 06:26:28 -07:00
|
|
|
WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier));
|
|
|
|
unregister_shrinker(&dev_priv->mm.shrinker);
|
2015-03-18 03:46:04 -06:00
|
|
|
}
|