2015-03-18 03:46:04 -06:00
|
|
|
/*
|
|
|
|
* Copyright © 2008-2015 Intel Corporation
|
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
|
|
* to deal in the Software without restriction, including without limitation
|
|
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice (including the next
|
|
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
|
|
* Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
|
|
* IN THE SOFTWARE.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/oom.h>
|
|
|
|
#include <linux/shmem_fs.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/swap.h>
|
|
|
|
#include <linux/pci.h>
|
|
|
|
#include <linux/dma-buf.h>
|
2016-04-04 07:46:43 -06:00
|
|
|
#include <linux/vmalloc.h>
|
2015-03-18 03:46:04 -06:00
|
|
|
#include <drm/drmP.h>
|
|
|
|
#include <drm/i915_drm.h>
|
|
|
|
|
|
|
|
#include "i915_drv.h"
|
|
|
|
#include "i915_trace.h"
|
|
|
|
|
|
|
|
static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
|
|
|
|
{
|
|
|
|
if (!mutex_is_locked(mutex))
|
|
|
|
return false;
|
|
|
|
|
2016-07-11 07:46:17 -06:00
|
|
|
#if defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_MUTEX_SPIN_ON_OWNER)
|
2015-03-18 03:46:04 -06:00
|
|
|
return mutex->owner == task;
|
|
|
|
#else
|
|
|
|
/* Since UP may be pre-empted, we cannot assume that we own the lock */
|
|
|
|
return false;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2016-08-04 00:52:26 -06:00
|
|
|
static bool any_vma_pinned(struct drm_i915_gem_object *obj)
|
2015-12-04 08:58:54 -07:00
|
|
|
{
|
|
|
|
struct i915_vma *vma;
|
|
|
|
|
2016-08-04 00:52:26 -06:00
|
|
|
list_for_each_entry(vma, &obj->vma_list, obj_link)
|
2016-08-04 09:32:32 -06:00
|
|
|
if (i915_vma_is_pinned(vma))
|
2016-08-04 00:52:26 -06:00
|
|
|
return true;
|
2015-12-04 08:58:54 -07:00
|
|
|
|
2016-08-04 00:52:26 -06:00
|
|
|
return false;
|
2015-12-04 08:58:54 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
static bool swap_available(void)
|
|
|
|
{
|
|
|
|
return get_nr_swap_pages() > 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool can_release_pages(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
2016-04-20 05:09:52 -06:00
|
|
|
/* Only shmemfs objects are backed by swap */
|
|
|
|
if (!obj->base.filp)
|
|
|
|
return false;
|
|
|
|
|
2015-12-04 08:58:54 -07:00
|
|
|
/* Only report true if by unbinding the object and putting its pages
|
|
|
|
* we can actually make forward progress towards freeing physical
|
|
|
|
* pages.
|
|
|
|
*
|
|
|
|
* If the pages are pinned for any other reason than being bound
|
|
|
|
* to the GPU, simply unbinding from the GPU is not going to succeed
|
|
|
|
* in releasing our pin count on the pages themselves.
|
|
|
|
*/
|
2016-08-04 00:52:26 -06:00
|
|
|
if (obj->pages_pin_count > obj->bind_count)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (any_vma_pinned(obj))
|
2015-12-04 08:58:54 -07:00
|
|
|
return false;
|
|
|
|
|
|
|
|
/* We can only return physical pages to the system if we can either
|
|
|
|
* discard the contents (because the user has marked them as being
|
|
|
|
* purgeable) or if we can move their contents out to swap.
|
|
|
|
*/
|
|
|
|
return swap_available() || obj->madv == I915_MADV_DONTNEED;
|
|
|
|
}
|
|
|
|
|
2015-03-18 07:47:59 -06:00
|
|
|
/**
|
|
|
|
* i915_gem_shrink - Shrink buffer object caches
|
|
|
|
* @dev_priv: i915 device
|
|
|
|
* @target: amount of memory to make available, in pages
|
|
|
|
* @flags: control flags for selecting cache types
|
|
|
|
*
|
|
|
|
* This function is the main interface to the shrinker. It will try to release
|
|
|
|
* up to @target pages of main memory backing storage from buffer objects.
|
|
|
|
* Selection of the specific caches can be done with @flags. This is e.g. useful
|
|
|
|
* when purgeable objects should be removed from caches preferentially.
|
|
|
|
*
|
|
|
|
* Note that it's not guaranteed that released amount is actually available as
|
|
|
|
* free system memory - the pages might still be in-used to due to other reasons
|
|
|
|
* (like cpu mmaps) or the mm core has reused them before we could grab them.
|
|
|
|
* Therefore code that needs to explicitly shrink buffer objects caches (e.g. to
|
|
|
|
* avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all().
|
|
|
|
*
|
|
|
|
* Also note that any kind of pinning (both per-vma address space pins and
|
|
|
|
* backing storage pins at the buffer object level) result in the shrinker code
|
|
|
|
* having to skip the object.
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* The number of pages of backing storage actually released.
|
|
|
|
*/
|
2015-03-18 03:46:04 -06:00
|
|
|
unsigned long
|
|
|
|
i915_gem_shrink(struct drm_i915_private *dev_priv,
|
2015-10-01 05:18:25 -06:00
|
|
|
unsigned long target, unsigned flags)
|
2015-03-18 03:46:04 -06:00
|
|
|
{
|
|
|
|
const struct {
|
|
|
|
struct list_head *list;
|
|
|
|
unsigned int bit;
|
|
|
|
} phases[] = {
|
|
|
|
{ &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
|
|
|
|
{ &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
|
|
|
|
{ NULL, 0 },
|
|
|
|
}, *phase;
|
|
|
|
unsigned long count = 0;
|
|
|
|
|
2015-10-01 05:18:26 -06:00
|
|
|
trace_i915_gem_shrink(dev_priv, target, flags);
|
2016-05-06 08:40:21 -06:00
|
|
|
i915_gem_retire_requests(dev_priv);
|
2015-10-01 05:18:26 -06:00
|
|
|
|
2016-05-02 02:40:28 -06:00
|
|
|
/*
|
|
|
|
* Unbinding of objects will require HW access; Let us not wake the
|
|
|
|
* device just to recover a little memory. If absolutely necessary,
|
|
|
|
* we will force the wake during oom-notifier.
|
|
|
|
*/
|
|
|
|
if ((flags & I915_SHRINK_BOUND) &&
|
|
|
|
!intel_runtime_pm_get_if_in_use(dev_priv))
|
|
|
|
flags &= ~I915_SHRINK_BOUND;
|
|
|
|
|
2015-03-18 03:46:04 -06:00
|
|
|
/*
|
|
|
|
* As we may completely rewrite the (un)bound list whilst unbinding
|
|
|
|
* (due to retiring requests) we have to strictly process only
|
|
|
|
* one element of the list at the time, and recheck the list
|
|
|
|
* on every iteration.
|
|
|
|
*
|
|
|
|
* In particular, we must hold a reference whilst removing the
|
|
|
|
* object as we may end up waiting for and/or retiring the objects.
|
|
|
|
* This might release the final reference (held by the active list)
|
|
|
|
* and result in the object being freed from under us. This is
|
|
|
|
* similar to the precautions the eviction code must take whilst
|
|
|
|
* removing objects.
|
|
|
|
*
|
|
|
|
* Also note that although these lists do not hold a reference to
|
|
|
|
* the object we can safely grab one here: The final object
|
|
|
|
* unreferencing and the bound_list are both protected by the
|
|
|
|
* dev->struct_mutex and so we won't ever be able to observe an
|
|
|
|
* object on the bound_list with a reference count equals 0.
|
|
|
|
*/
|
|
|
|
for (phase = phases; phase->list; phase++) {
|
|
|
|
struct list_head still_in_list;
|
2016-07-26 05:01:51 -06:00
|
|
|
struct drm_i915_gem_object *obj;
|
2015-03-18 03:46:04 -06:00
|
|
|
|
|
|
|
if ((flags & phase->bit) == 0)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
INIT_LIST_HEAD(&still_in_list);
|
2016-07-26 05:01:51 -06:00
|
|
|
while (count < target &&
|
|
|
|
(obj = list_first_entry_or_null(phase->list,
|
|
|
|
typeof(*obj),
|
|
|
|
global_list))) {
|
2015-03-18 03:46:04 -06:00
|
|
|
list_move_tail(&obj->global_list, &still_in_list);
|
|
|
|
|
|
|
|
if (flags & I915_SHRINK_PURGEABLE &&
|
|
|
|
obj->madv != I915_MADV_DONTNEED)
|
|
|
|
continue;
|
|
|
|
|
2016-04-08 05:11:12 -06:00
|
|
|
if (flags & I915_SHRINK_VMAPS &&
|
|
|
|
!is_vmalloc_addr(obj->mapping))
|
|
|
|
continue;
|
|
|
|
|
2016-08-04 09:32:39 -06:00
|
|
|
if ((flags & I915_SHRINK_ACTIVE) == 0 &&
|
|
|
|
i915_gem_object_is_active(obj))
|
2015-10-01 05:18:29 -06:00
|
|
|
continue;
|
|
|
|
|
2015-12-04 08:58:54 -07:00
|
|
|
if (!can_release_pages(obj))
|
|
|
|
continue;
|
|
|
|
|
2016-07-20 06:31:52 -06:00
|
|
|
i915_gem_object_get(obj);
|
2015-03-18 03:46:04 -06:00
|
|
|
|
|
|
|
/* For the unbound phase, this should be a no-op! */
|
2016-08-04 00:52:27 -06:00
|
|
|
i915_gem_object_unbind(obj);
|
2015-03-18 03:46:04 -06:00
|
|
|
if (i915_gem_object_put_pages(obj) == 0)
|
|
|
|
count += obj->base.size >> PAGE_SHIFT;
|
|
|
|
|
2016-07-20 06:31:53 -06:00
|
|
|
i915_gem_object_put(obj);
|
2015-03-18 03:46:04 -06:00
|
|
|
}
|
|
|
|
list_splice(&still_in_list, phase->list);
|
|
|
|
}
|
|
|
|
|
2016-05-02 02:40:28 -06:00
|
|
|
if (flags & I915_SHRINK_BOUND)
|
|
|
|
intel_runtime_pm_put(dev_priv);
|
|
|
|
|
2016-05-06 08:40:21 -06:00
|
|
|
i915_gem_retire_requests(dev_priv);
|
drm/i915: Enable lockless lookup of request tracking via RCU
If we enable RCU for the requests (providing a grace period where we can
inspect a "dead" request before it is freed), we can allow callers to
carefully perform lockless lookup of an active request.
However, by enabling deferred freeing of requests, we can potentially
hog a lot of memory when dealing with tens of thousands of requests per
second - with a quick insertion of a synchronize_rcu() inside our
shrinker callback, that issue disappears.
v2: Currently, it is our responsibility to handle reclaim i.e. to avoid
hogging memory with the delayed slab frees. At the moment, we wait for a
grace period in the shrinker, and block for all RCU callbacks on oom.
Suggested alternatives focus on flushing our RCU callback when we have a
certain number of outstanding request frees, and blocking on that flush
after a second high watermark. (So rather than wait for the system to
run out of memory, we stop issuing requests - both are nondeterministic.)
Paul E. McKenney wrote:
Another approach is synchronize_rcu() after some largish number of
requests. The advantage of this approach is that it throttles the
production of callbacks at the source. The corresponding disadvantage
is that it slows things up.
Another approach is to use call_rcu(), but if the previous call_rcu()
is still in flight, block waiting for it. Yet another approach is
the get_state_synchronize_rcu() / cond_synchronize_rcu() pair. The
idea is to do something like this:
cond_synchronize_rcu(cookie);
cookie = get_state_synchronize_rcu();
You would of course do an initial get_state_synchronize_rcu() to
get things going. This would not block unless there was less than
one grace period's worth of time between invocations. But this
assumes a busy system, where there is almost always a grace period
in flight. But you can make that happen as follows:
cond_synchronize_rcu(cookie);
cookie = get_state_synchronize_rcu();
call_rcu(&my_rcu_head, noop_function);
Note that you need additional code to make sure that the old callback
has completed before doing a new one. Setting and clearing a flag
with appropriate memory ordering control suffices (e.g,. smp_load_acquire()
and smp_store_release()).
v3: More comments on compiler and processor order of operations within
the RCU lookup and discover we can use rcu_access_pointer() here instead.
v4: Wrap i915_gem_active_get_rcu() to take the rcu_read_lock itself.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Maarten Lankhorst <maarten.lankhorst@linux.intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Link: http://patchwork.freedesktop.org/patch/msgid/1470324762-2545-25-git-send-email-chris@chris-wilson.co.uk
2016-08-04 09:32:41 -06:00
|
|
|
/* expedite the RCU grace period to free some request slabs */
|
|
|
|
synchronize_rcu_expedited();
|
2015-10-01 05:18:27 -06:00
|
|
|
|
2015-03-18 03:46:04 -06:00
|
|
|
return count;
|
|
|
|
}
|
|
|
|
|
2015-03-18 07:47:59 -06:00
|
|
|
/**
|
2015-10-06 06:47:55 -06:00
|
|
|
* i915_gem_shrink_all - Shrink buffer object caches completely
|
2015-03-18 07:47:59 -06:00
|
|
|
* @dev_priv: i915 device
|
|
|
|
*
|
|
|
|
* This is a simple wraper around i915_gem_shrink() to aggressively shrink all
|
|
|
|
* caches completely. It also first waits for and retires all outstanding
|
|
|
|
* requests to also be able to release backing storage for active objects.
|
|
|
|
*
|
|
|
|
* This should only be used in code to intentionally quiescent the gpu or as a
|
|
|
|
* last-ditch effort when memory seems to have run out.
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* The number of pages of backing storage actually released.
|
|
|
|
*/
|
2015-03-18 03:46:04 -06:00
|
|
|
unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
|
|
|
|
{
|
drm/i915: Enable lockless lookup of request tracking via RCU
If we enable RCU for the requests (providing a grace period where we can
inspect a "dead" request before it is freed), we can allow callers to
carefully perform lockless lookup of an active request.
However, by enabling deferred freeing of requests, we can potentially
hog a lot of memory when dealing with tens of thousands of requests per
second - with a quick insertion of a synchronize_rcu() inside our
shrinker callback, that issue disappears.
v2: Currently, it is our responsibility to handle reclaim i.e. to avoid
hogging memory with the delayed slab frees. At the moment, we wait for a
grace period in the shrinker, and block for all RCU callbacks on oom.
Suggested alternatives focus on flushing our RCU callback when we have a
certain number of outstanding request frees, and blocking on that flush
after a second high watermark. (So rather than wait for the system to
run out of memory, we stop issuing requests - both are nondeterministic.)
Paul E. McKenney wrote:
Another approach is synchronize_rcu() after some largish number of
requests. The advantage of this approach is that it throttles the
production of callbacks at the source. The corresponding disadvantage
is that it slows things up.
Another approach is to use call_rcu(), but if the previous call_rcu()
is still in flight, block waiting for it. Yet another approach is
the get_state_synchronize_rcu() / cond_synchronize_rcu() pair. The
idea is to do something like this:
cond_synchronize_rcu(cookie);
cookie = get_state_synchronize_rcu();
You would of course do an initial get_state_synchronize_rcu() to
get things going. This would not block unless there was less than
one grace period's worth of time between invocations. But this
assumes a busy system, where there is almost always a grace period
in flight. But you can make that happen as follows:
cond_synchronize_rcu(cookie);
cookie = get_state_synchronize_rcu();
call_rcu(&my_rcu_head, noop_function);
Note that you need additional code to make sure that the old callback
has completed before doing a new one. Setting and clearing a flag
with appropriate memory ordering control suffices (e.g,. smp_load_acquire()
and smp_store_release()).
v3: More comments on compiler and processor order of operations within
the RCU lookup and discover we can use rcu_access_pointer() here instead.
v4: Wrap i915_gem_active_get_rcu() to take the rcu_read_lock itself.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Maarten Lankhorst <maarten.lankhorst@linux.intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Link: http://patchwork.freedesktop.org/patch/msgid/1470324762-2545-25-git-send-email-chris@chris-wilson.co.uk
2016-08-04 09:32:41 -06:00
|
|
|
unsigned long freed;
|
|
|
|
|
|
|
|
freed = i915_gem_shrink(dev_priv, -1UL,
|
|
|
|
I915_SHRINK_BOUND |
|
|
|
|
I915_SHRINK_UNBOUND |
|
|
|
|
I915_SHRINK_ACTIVE);
|
|
|
|
rcu_barrier(); /* wait until our RCU delayed slab frees are completed */
|
|
|
|
|
|
|
|
return freed;
|
2015-03-18 03:46:04 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
|
|
|
|
{
|
|
|
|
if (!mutex_trylock(&dev->struct_mutex)) {
|
|
|
|
if (!mutex_is_locked_by(&dev->struct_mutex, current))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
*unlock = false;
|
|
|
|
} else
|
|
|
|
*unlock = true;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned long
|
|
|
|
i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv =
|
|
|
|
container_of(shrinker, struct drm_i915_private, mm.shrinker);
|
2016-07-05 03:40:23 -06:00
|
|
|
struct drm_device *dev = &dev_priv->drm;
|
2015-03-18 03:46:04 -06:00
|
|
|
struct drm_i915_gem_object *obj;
|
|
|
|
unsigned long count;
|
|
|
|
bool unlock;
|
|
|
|
|
|
|
|
if (!i915_gem_shrinker_lock(dev, &unlock))
|
|
|
|
return 0;
|
|
|
|
|
2016-07-01 10:23:10 -06:00
|
|
|
i915_gem_retire_requests(dev_priv);
|
|
|
|
|
2015-03-18 03:46:04 -06:00
|
|
|
count = 0;
|
|
|
|
list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
|
2016-04-04 07:46:41 -06:00
|
|
|
if (can_release_pages(obj))
|
2015-03-18 03:46:04 -06:00
|
|
|
count += obj->base.size >> PAGE_SHIFT;
|
|
|
|
|
|
|
|
list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
|
2016-08-04 09:32:39 -06:00
|
|
|
if (!i915_gem_object_is_active(obj) && can_release_pages(obj))
|
2015-03-18 03:46:04 -06:00
|
|
|
count += obj->base.size >> PAGE_SHIFT;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (unlock)
|
|
|
|
mutex_unlock(&dev->struct_mutex);
|
|
|
|
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned long
|
|
|
|
i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv =
|
|
|
|
container_of(shrinker, struct drm_i915_private, mm.shrinker);
|
2016-07-05 03:40:23 -06:00
|
|
|
struct drm_device *dev = &dev_priv->drm;
|
2015-03-18 03:46:04 -06:00
|
|
|
unsigned long freed;
|
|
|
|
bool unlock;
|
|
|
|
|
|
|
|
if (!i915_gem_shrinker_lock(dev, &unlock))
|
|
|
|
return SHRINK_STOP;
|
|
|
|
|
|
|
|
freed = i915_gem_shrink(dev_priv,
|
|
|
|
sc->nr_to_scan,
|
|
|
|
I915_SHRINK_BOUND |
|
|
|
|
I915_SHRINK_UNBOUND |
|
|
|
|
I915_SHRINK_PURGEABLE);
|
|
|
|
if (freed < sc->nr_to_scan)
|
|
|
|
freed += i915_gem_shrink(dev_priv,
|
|
|
|
sc->nr_to_scan - freed,
|
|
|
|
I915_SHRINK_BOUND |
|
|
|
|
I915_SHRINK_UNBOUND);
|
|
|
|
if (unlock)
|
|
|
|
mutex_unlock(&dev->struct_mutex);
|
|
|
|
|
|
|
|
return freed;
|
|
|
|
}
|
|
|
|
|
2016-04-05 03:22:25 -06:00
|
|
|
struct shrinker_lock_uninterruptible {
|
|
|
|
bool was_interruptible;
|
|
|
|
bool unlock;
|
|
|
|
};
|
|
|
|
|
|
|
|
static bool
|
|
|
|
i915_gem_shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv,
|
|
|
|
struct shrinker_lock_uninterruptible *slu,
|
|
|
|
int timeout_ms)
|
|
|
|
{
|
2016-08-05 03:14:13 -06:00
|
|
|
unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms);
|
|
|
|
|
|
|
|
do {
|
2016-09-09 07:11:49 -06:00
|
|
|
if (i915_gem_wait_for_idle(dev_priv, 0) == 0 &&
|
2016-08-05 03:14:13 -06:00
|
|
|
i915_gem_shrinker_lock(&dev_priv->drm, &slu->unlock))
|
|
|
|
break;
|
2016-04-05 03:22:25 -06:00
|
|
|
|
|
|
|
schedule_timeout_killable(1);
|
|
|
|
if (fatal_signal_pending(current))
|
|
|
|
return false;
|
2016-08-05 03:14:13 -06:00
|
|
|
|
|
|
|
if (time_after(jiffies, timeout)) {
|
2016-04-05 03:22:25 -06:00
|
|
|
pr_err("Unable to lock GPU to purge memory.\n");
|
|
|
|
return false;
|
|
|
|
}
|
2016-08-05 03:14:13 -06:00
|
|
|
} while (1);
|
2016-04-05 03:22:25 -06:00
|
|
|
|
|
|
|
slu->was_interruptible = dev_priv->mm.interruptible;
|
|
|
|
dev_priv->mm.interruptible = false;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
i915_gem_shrinker_unlock_uninterruptible(struct drm_i915_private *dev_priv,
|
|
|
|
struct shrinker_lock_uninterruptible *slu)
|
|
|
|
{
|
|
|
|
dev_priv->mm.interruptible = slu->was_interruptible;
|
|
|
|
if (slu->unlock)
|
2016-07-05 03:40:23 -06:00
|
|
|
mutex_unlock(&dev_priv->drm.struct_mutex);
|
2016-04-05 03:22:25 -06:00
|
|
|
}
|
|
|
|
|
2015-03-18 03:46:04 -06:00
|
|
|
static int
|
|
|
|
i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv =
|
|
|
|
container_of(nb, struct drm_i915_private, mm.oom_notifier);
|
2016-04-05 03:22:25 -06:00
|
|
|
struct shrinker_lock_uninterruptible slu;
|
2015-03-18 03:46:04 -06:00
|
|
|
struct drm_i915_gem_object *obj;
|
2016-04-20 05:09:51 -06:00
|
|
|
unsigned long unevictable, bound, unbound, freed_pages;
|
2015-03-18 03:46:04 -06:00
|
|
|
|
2016-04-05 03:22:25 -06:00
|
|
|
if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000))
|
2015-03-18 03:46:04 -06:00
|
|
|
return NOTIFY_DONE;
|
|
|
|
|
2016-05-02 02:40:29 -06:00
|
|
|
intel_runtime_pm_get(dev_priv);
|
2015-03-18 03:46:04 -06:00
|
|
|
freed_pages = i915_gem_shrink_all(dev_priv);
|
2016-05-02 02:40:29 -06:00
|
|
|
intel_runtime_pm_put(dev_priv);
|
2015-03-18 03:46:04 -06:00
|
|
|
|
|
|
|
/* Because we may be allocating inside our own driver, we cannot
|
|
|
|
* assert that there are no objects with pinned pages that are not
|
|
|
|
* being pointed to by hardware.
|
|
|
|
*/
|
2016-04-20 05:09:51 -06:00
|
|
|
unbound = bound = unevictable = 0;
|
2015-03-18 03:46:04 -06:00
|
|
|
list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
|
2016-04-20 05:09:51 -06:00
|
|
|
if (!can_release_pages(obj))
|
|
|
|
unevictable += obj->base.size >> PAGE_SHIFT;
|
2015-03-18 03:46:04 -06:00
|
|
|
else
|
2016-04-20 05:09:51 -06:00
|
|
|
unbound += obj->base.size >> PAGE_SHIFT;
|
2015-03-18 03:46:04 -06:00
|
|
|
}
|
|
|
|
list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
|
2016-04-20 05:09:51 -06:00
|
|
|
if (!can_release_pages(obj))
|
|
|
|
unevictable += obj->base.size >> PAGE_SHIFT;
|
2015-03-18 03:46:04 -06:00
|
|
|
else
|
2016-04-20 05:09:51 -06:00
|
|
|
bound += obj->base.size >> PAGE_SHIFT;
|
2015-03-18 03:46:04 -06:00
|
|
|
}
|
|
|
|
|
2016-04-05 03:22:25 -06:00
|
|
|
i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu);
|
2015-03-18 03:46:04 -06:00
|
|
|
|
|
|
|
if (freed_pages || unbound || bound)
|
2016-04-20 05:09:51 -06:00
|
|
|
pr_info("Purging GPU memory, %lu pages freed, "
|
|
|
|
"%lu pages still pinned.\n",
|
|
|
|
freed_pages, unevictable);
|
2015-03-18 03:46:04 -06:00
|
|
|
if (unbound || bound)
|
2016-04-20 05:09:51 -06:00
|
|
|
pr_err("%lu and %lu pages still available in the "
|
2015-03-18 03:46:04 -06:00
|
|
|
"bound and unbound GPU page lists.\n",
|
|
|
|
bound, unbound);
|
|
|
|
|
|
|
|
*(unsigned long *)ptr += freed_pages;
|
|
|
|
return NOTIFY_DONE;
|
|
|
|
}
|
|
|
|
|
2016-04-04 07:46:43 -06:00
|
|
|
static int
|
|
|
|
i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv =
|
|
|
|
container_of(nb, struct drm_i915_private, mm.vmap_notifier);
|
2016-04-05 03:22:25 -06:00
|
|
|
struct shrinker_lock_uninterruptible slu;
|
2016-04-28 02:56:39 -06:00
|
|
|
struct i915_vma *vma, *next;
|
|
|
|
unsigned long freed_pages = 0;
|
|
|
|
int ret;
|
2016-04-04 07:46:43 -06:00
|
|
|
|
2016-04-05 03:22:25 -06:00
|
|
|
if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000))
|
2016-04-04 07:46:43 -06:00
|
|
|
return NOTIFY_DONE;
|
|
|
|
|
2016-04-28 02:56:39 -06:00
|
|
|
/* Force everything onto the inactive lists */
|
2016-09-09 07:11:50 -06:00
|
|
|
ret = i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED);
|
2016-04-28 02:56:39 -06:00
|
|
|
if (ret)
|
|
|
|
goto out;
|
|
|
|
|
2016-05-02 02:40:29 -06:00
|
|
|
intel_runtime_pm_get(dev_priv);
|
2016-04-28 02:56:39 -06:00
|
|
|
freed_pages += i915_gem_shrink(dev_priv, -1UL,
|
|
|
|
I915_SHRINK_BOUND |
|
|
|
|
I915_SHRINK_UNBOUND |
|
|
|
|
I915_SHRINK_ACTIVE |
|
|
|
|
I915_SHRINK_VMAPS);
|
2016-05-02 02:40:29 -06:00
|
|
|
intel_runtime_pm_put(dev_priv);
|
2016-04-28 02:56:39 -06:00
|
|
|
|
|
|
|
/* We also want to clear any cached iomaps as they wrap vmap */
|
|
|
|
list_for_each_entry_safe(vma, next,
|
|
|
|
&dev_priv->ggtt.base.inactive_list, vm_link) {
|
|
|
|
unsigned long count = vma->node.size >> PAGE_SHIFT;
|
|
|
|
if (vma->iomap && i915_vma_unbind(vma) == 0)
|
|
|
|
freed_pages += count;
|
|
|
|
}
|
2016-04-04 07:46:43 -06:00
|
|
|
|
2016-04-28 02:56:39 -06:00
|
|
|
out:
|
2016-04-05 03:22:25 -06:00
|
|
|
i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu);
|
2016-04-04 07:46:43 -06:00
|
|
|
|
|
|
|
*(unsigned long *)ptr += freed_pages;
|
|
|
|
return NOTIFY_DONE;
|
|
|
|
}
|
|
|
|
|
2015-03-18 07:47:59 -06:00
|
|
|
/**
|
|
|
|
* i915_gem_shrinker_init - Initialize i915 shrinker
|
|
|
|
* @dev_priv: i915 device
|
|
|
|
*
|
|
|
|
* This function registers and sets up the i915 shrinker and OOM handler.
|
|
|
|
*/
|
2015-03-18 03:46:04 -06:00
|
|
|
void i915_gem_shrinker_init(struct drm_i915_private *dev_priv)
|
|
|
|
{
|
|
|
|
dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
|
|
|
|
dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count;
|
|
|
|
dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS;
|
2016-01-19 06:26:28 -07:00
|
|
|
WARN_ON(register_shrinker(&dev_priv->mm.shrinker));
|
2015-03-18 03:46:04 -06:00
|
|
|
|
|
|
|
dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
|
2016-01-19 06:26:28 -07:00
|
|
|
WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier));
|
2016-04-04 07:46:43 -06:00
|
|
|
|
|
|
|
dev_priv->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap;
|
|
|
|
WARN_ON(register_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
|
2016-01-19 06:26:28 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* i915_gem_shrinker_cleanup - Clean up i915 shrinker
|
|
|
|
* @dev_priv: i915 device
|
|
|
|
*
|
|
|
|
* This function unregisters the i915 shrinker and OOM handler.
|
|
|
|
*/
|
|
|
|
void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv)
|
|
|
|
{
|
2016-04-04 07:46:43 -06:00
|
|
|
WARN_ON(unregister_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
|
2016-01-19 06:26:28 -07:00
|
|
|
WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier));
|
|
|
|
unregister_shrinker(&dev_priv->mm.shrinker);
|
2015-03-18 03:46:04 -06:00
|
|
|
}
|