![Chris Wilson](/assets/img/avatar_default.png)
The rcu_barrier() takes the cpu_hotplug mutex which itself is not
reclaim-safe, and so rcu_barrier() is illegal from inside the shrinker.
[ 309.661373] =========================================================
[ 309.661376] [ INFO: possible irq lock inversion dependency detected ]
[ 309.661380] 4.11.0-rc1-CI-CI_DRM_2333+ #1 Tainted: G W
[ 309.661383] ---------------------------------------------------------
[ 309.661386] gem_exec_gttfil/6435 just changed the state of lock:
[ 309.661389] (rcu_preempt_state.barrier_mutex){+.+.-.}, at: [<ffffffff81100731>] _rcu_barrier+0x31/0x160
[ 309.661399] but this lock took another, RECLAIM_FS-unsafe lock in the past:
[ 309.661402] (cpu_hotplug.lock){+.+.+.}
[ 309.661404]
and interrupts could create inverse lock ordering between them.
[ 309.661410]
other info that might help us debug this:
[ 309.661414] Possible interrupt unsafe locking scenario:
[ 309.661417] CPU0 CPU1
[ 309.661419] ---- ----
[ 309.661421] lock(cpu_hotplug.lock);
[ 309.661425] local_irq_disable();
[ 309.661432] lock(rcu_preempt_state.barrier_mutex);
[ 309.661441] lock(cpu_hotplug.lock);
[ 309.661446] <Interrupt>
[ 309.661448] lock(rcu_preempt_state.barrier_mutex);
[ 309.661453]
*** DEADLOCK ***
[ 309.661460] 4 locks held by gem_exec_gttfil/6435:
[ 309.661464] #0: (sb_writers#10){.+.+.+}, at: [<ffffffff8120d83d>] vfs_write+0x17d/0x1f0
[ 309.661475] #1: (debugfs_srcu){......}, at: [<ffffffff81320491>] debugfs_use_file_start+0x41/0xa0
[ 309.661486] #2: (&attr->mutex){+.+.+.}, at: [<ffffffff8123a3e7>] simple_attr_write+0x37/0xe0
[ 309.661495] #3: (&dev->struct_mutex){+.+.+.}, at: [<ffffffffa0091b4a>] i915_drop_caches_set+0x3a/0x150 [i915]
[ 309.661540]
the shortest dependencies between 2nd lock and 1st lock:
[ 309.661547] -> (cpu_hotplug.lock){+.+.+.} ops: 829 {
[ 309.661553] HARDIRQ-ON-W at:
[ 309.661560] __lock_acquire+0x5e5/0x1b50
[ 309.661565] lock_acquire+0xc9/0x220
[ 309.661572] __mutex_lock+0x6e/0x990
[ 309.661576] mutex_lock_nested+0x16/0x20
[ 309.661583] get_online_cpus+0x61/0x80
[ 309.661590] kmem_cache_create+0x25/0x1d0
[ 309.661596] debug_objects_mem_init+0x30/0x249
[ 309.661602] start_kernel+0x341/0x3fe
[ 309.661607] x86_64_start_reservations+0x2a/0x2c
[ 309.661612] x86_64_start_kernel+0x173/0x186
[ 309.661619] verify_cpu+0x0/0xfc
[ 309.661622] SOFTIRQ-ON-W at:
[ 309.661627] __lock_acquire+0x611/0x1b50
[ 309.661632] lock_acquire+0xc9/0x220
[ 309.661636] __mutex_lock+0x6e/0x990
[ 309.661641] mutex_lock_nested+0x16/0x20
[ 309.661646] get_online_cpus+0x61/0x80
[ 309.661650] kmem_cache_create+0x25/0x1d0
[ 309.661655] debug_objects_mem_init+0x30/0x249
[ 309.661660] start_kernel+0x341/0x3fe
[ 309.661664] x86_64_start_reservations+0x2a/0x2c
[ 309.661669] x86_64_start_kernel+0x173/0x186
[ 309.661674] verify_cpu+0x0/0xfc
[ 309.661677] RECLAIM_FS-ON-W at:
[ 309.661682] mark_held_locks+0x6f/0xa0
[ 309.661687] lockdep_trace_alloc+0xb3/0x100
[ 309.661693] kmem_cache_alloc_trace+0x31/0x2e0
[ 309.661699] __smpboot_create_thread.part.1+0x27/0xe0
[ 309.661704] smpboot_create_threads+0x61/0x90
[ 309.661709] cpuhp_invoke_callback+0x9c/0x8a0
[ 309.661713] cpuhp_up_callbacks+0x31/0xb0
[ 309.661718] _cpu_up+0x7a/0xc0
[ 309.661723] do_cpu_up+0x5f/0x80
[ 309.661727] cpu_up+0xe/0x10
[ 309.661734] smp_init+0x71/0xb3
[ 309.661738] kernel_init_freeable+0x94/0x19e
[ 309.661743] kernel_init+0x9/0xf0
[ 309.661748] ret_from_fork+0x2e/0x40
[ 309.661752] INITIAL USE at:
[ 309.661757] __lock_acquire+0x234/0x1b50
[ 309.661761] lock_acquire+0xc9/0x220
[ 309.661766] __mutex_lock+0x6e/0x990
[ 309.661771] mutex_lock_nested+0x16/0x20
[ 309.661775] get_online_cpus+0x61/0x80
[ 309.661780] __cpuhp_setup_state+0x44/0x170
[ 309.661785] page_alloc_init+0x23/0x3a
[ 309.661790] start_kernel+0x124/0x3fe
[ 309.661794] x86_64_start_reservations+0x2a/0x2c
[ 309.661799] x86_64_start_kernel+0x173/0x186
[ 309.661804] verify_cpu+0x0/0xfc
[ 309.661807] }
[ 309.661813] ... key at: [<ffffffff81e37690>] cpu_hotplug+0xb0/0x100
[ 309.661817] ... acquired at:
[ 309.661821] lock_acquire+0xc9/0x220
[ 309.661825] __mutex_lock+0x6e/0x990
[ 309.661829] mutex_lock_nested+0x16/0x20
[ 309.661833] get_online_cpus+0x61/0x80
[ 309.661837] _rcu_barrier+0x9f/0x160
[ 309.661841] rcu_barrier+0x10/0x20
[ 309.661847] netdev_run_todo+0x5f/0x310
[ 309.661852] rtnl_unlock+0x9/0x10
[ 309.661856] default_device_exit_batch+0x133/0x150
[ 309.661862] ops_exit_list.isra.0+0x4d/0x60
[ 309.661866] cleanup_net+0x1d8/0x2c0
[ 309.661872] process_one_work+0x1f4/0x6d0
[ 309.661876] worker_thread+0x49/0x4a0
[ 309.661881] kthread+0x107/0x140
[ 309.661884] ret_from_fork+0x2e/0x40
[ 309.661890] -> (rcu_preempt_state.barrier_mutex){+.+.-.} ops: 179 {
[ 309.661896] HARDIRQ-ON-W at:
[ 309.661901] __lock_acquire+0x5e5/0x1b50
[ 309.661905] lock_acquire+0xc9/0x220
[ 309.661910] __mutex_lock+0x6e/0x990
[ 309.661914] mutex_lock_nested+0x16/0x20
[ 309.661919] _rcu_barrier+0x31/0x160
[ 309.661923] rcu_barrier+0x10/0x20
[ 309.661928] netdev_run_todo+0x5f/0x310
[ 309.661932] rtnl_unlock+0x9/0x10
[ 309.661936] default_device_exit_batch+0x133/0x150
[ 309.661941] ops_exit_list.isra.0+0x4d/0x60
[ 309.661946] cleanup_net+0x1d8/0x2c0
[ 309.661951] process_one_work+0x1f4/0x6d0
[ 309.661955] worker_thread+0x49/0x4a0
[ 309.661960] kthread+0x107/0x140
[ 309.661964] ret_from_fork+0x2e/0x40
[ 309.661968] SOFTIRQ-ON-W at:
[ 309.661972] __lock_acquire+0x611/0x1b50
[ 309.661977] lock_acquire+0xc9/0x220
[ 309.661981] __mutex_lock+0x6e/0x990
[ 309.661986] mutex_lock_nested+0x16/0x20
[ 309.661990] _rcu_barrier+0x31/0x160
[ 309.661995] rcu_barrier+0x10/0x20
[ 309.661999] netdev_run_todo+0x5f/0x310
[ 309.662003] rtnl_unlock+0x9/0x10
[ 309.662008] default_device_exit_batch+0x133/0x150
[ 309.662013] ops_exit_list.isra.0+0x4d/0x60
[ 309.662017] cleanup_net+0x1d8/0x2c0
[ 309.662022] process_one_work+0x1f4/0x6d0
[ 309.662027] worker_thread+0x49/0x4a0
[ 309.662031] kthread+0x107/0x140
[ 309.662035] ret_from_fork+0x2e/0x40
[ 309.662039] IN-RECLAIM_FS-W at:
[ 309.662043] __lock_acquire+0x638/0x1b50
[ 309.662048] lock_acquire+0xc9/0x220
[ 309.662053] __mutex_lock+0x6e/0x990
[ 309.662058] mutex_lock_nested+0x16/0x20
[ 309.662062] _rcu_barrier+0x31/0x160
[ 309.662067] rcu_barrier+0x10/0x20
[ 309.662089] i915_gem_shrink_all+0x33/0x40 [i915]
[ 309.662109] i915_drop_caches_set+0x141/0x150 [i915]
[ 309.662114] simple_attr_write+0xc7/0xe0
[ 309.662119] full_proxy_write+0x4f/0x70
[ 309.662124] __vfs_write+0x23/0x120
[ 309.662128] vfs_write+0xc6/0x1f0
[ 309.662133] SyS_write+0x44/0xb0
[ 309.662138] entry_SYSCALL_64_fastpath+0x1c/0xb1
[ 309.662142] INITIAL USE at:
[ 309.662147] __lock_acquire+0x234/0x1b50
[ 309.662151] lock_acquire+0xc9/0x220
[ 309.662156] __mutex_lock+0x6e/0x990
[ 309.662160] mutex_lock_nested+0x16/0x20
[ 309.662165] _rcu_barrier+0x31/0x160
[ 309.662169] rcu_barrier+0x10/0x20
[ 309.662174] netdev_run_todo+0x5f/0x310
[ 309.662178] rtnl_unlock+0x9/0x10
[ 309.662183] default_device_exit_batch+0x133/0x150
[ 309.662188] ops_exit_list.isra.0+0x4d/0x60
[ 309.662192] cleanup_net+0x1d8/0x2c0
[ 309.662197] process_one_work+0x1f4/0x6d0
[ 309.662202] worker_thread+0x49/0x4a0
[ 309.662206] kthread+0x107/0x140
[ 309.662210] ret_from_fork+0x2e/0x40
[ 309.662214] }
[ 309.662220] ... key at: [<ffffffff81e4e1c8>] rcu_preempt_state+0x508/0x780
[ 309.662225] ... acquired at:
[ 309.662229] check_usage_forwards+0x12b/0x130
[ 309.662233] mark_lock+0x360/0x6f0
[ 309.662237] __lock_acquire+0x638/0x1b50
[ 309.662241] lock_acquire+0xc9/0x220
[ 309.662245] __mutex_lock+0x6e/0x990
[ 309.662249] mutex_lock_nested+0x16/0x20
[ 309.662253] _rcu_barrier+0x31/0x160
[ 309.662257] rcu_barrier+0x10/0x20
[ 309.662279] i915_gem_shrink_all+0x33/0x40 [i915]
[ 309.662298] i915_drop_caches_set+0x141/0x150 [i915]
[ 309.662303] simple_attr_write+0xc7/0xe0
[ 309.662307] full_proxy_write+0x4f/0x70
[ 309.662311] __vfs_write+0x23/0x120
[ 309.662315] vfs_write+0xc6/0x1f0
[ 309.662319] SyS_write+0x44/0xb0
[ 309.662323] entry_SYSCALL_64_fastpath+0x1c/0xb1
[ 309.662329]
stack backtrace:
[ 309.662335] CPU: 1 PID: 6435 Comm: gem_exec_gttfil Tainted: G W 4.11.0-rc1-CI-CI_DRM_2333+ #1
[ 309.662342] Hardware name: Hewlett-Packard HP Compaq 8100 Elite SFF PC/304Ah, BIOS 786H1 v01.13 07/14/2011
[ 309.662348] Call Trace:
[ 309.662354] dump_stack+0x67/0x92
[ 309.662359] print_irq_inversion_bug.part.19+0x1a4/0x1b0
[ 309.662365] check_usage_forwards+0x12b/0x130
[ 309.662369] mark_lock+0x360/0x6f0
[ 309.662374] ? print_shortest_lock_dependencies+0x1a0/0x1a0
[ 309.662379] __lock_acquire+0x638/0x1b50
[ 309.662383] ? __mutex_unlock_slowpath+0x3e/0x2e0
[ 309.662388] ? trace_hardirqs_on+0xd/0x10
[ 309.662392] ? _rcu_barrier+0x31/0x160
[ 309.662396] lock_acquire+0xc9/0x220
[ 309.662400] ? _rcu_barrier+0x31/0x160
[ 309.662404] ? _rcu_barrier+0x31/0x160
[ 309.662409] __mutex_lock+0x6e/0x990
[ 309.662412] ? _rcu_barrier+0x31/0x160
[ 309.662416] ? _rcu_barrier+0x31/0x160
[ 309.662421] ? synchronize_rcu_expedited+0x35/0xb0
[ 309.662426] ? _raw_spin_unlock_irqrestore+0x52/0x60
[ 309.662434] mutex_lock_nested+0x16/0x20
[ 309.662438] _rcu_barrier+0x31/0x160
[ 309.662442] rcu_barrier+0x10/0x20
[ 309.662464] i915_gem_shrink_all+0x33/0x40 [i915]
[ 309.662484] i915_drop_caches_set+0x141/0x150 [i915]
[ 309.662489] simple_attr_write+0xc7/0xe0
[ 309.662494] full_proxy_write+0x4f/0x70
[ 309.662498] __vfs_write+0x23/0x120
[ 309.662503] ? rcu_read_lock_sched_held+0x75/0x80
[ 309.662507] ? rcu_sync_lockdep_assert+0x2a/0x50
[ 309.662512] ? __sb_start_write+0x102/0x210
[ 309.662516] ? vfs_write+0x17d/0x1f0
[ 309.662520] vfs_write+0xc6/0x1f0
[ 309.662524] ? trace_hardirqs_on_caller+0xe7/0x200
[ 309.662529] SyS_write+0x44/0xb0
[ 309.662533] entry_SYSCALL_64_fastpath+0x1c/0xb1
[ 309.662537] RIP: 0033:0x7f507eac24a0
[ 309.662541] RSP: 002b:00007fffda8720e8 EFLAGS: 00000246 ORIG_RAX: 0000000000000001
[ 309.662548] RAX: ffffffffffffffda RBX: ffffffff81482bd3 RCX: 00007f507eac24a0
[ 309.662552] RDX: 0000000000000005 RSI: 00007fffda8720f0 RDI: 0000000000000005
[ 309.662557] RBP: ffffc9000048bf88 R08: 0000000000000000 R09: 000000000000002c
[ 309.662561] R10: 0000000000000014 R11: 0000000000000246 R12: 00007fffda872230
[ 309.662566] R13: 00007fffda872228 R14: 0000000000000201 R15: 00007fffda8720f0
[ 309.662572] ? __this_cpu_preempt_check+0x13/0x20
Fixes: 0eafec6d32
("drm/i915: Enable lockless lookup of request tracking via RCU")
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=100192
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: <stable@vger.kernel.org> # v4.9+
Link: http://patchwork.freedesktop.org/patch/msgid/20170314115019.18127-1-chris@chris-wilson.co.uk
Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
500 lines
15 KiB
C
500 lines
15 KiB
C
/*
|
|
* Copyright © 2008-2015 Intel Corporation
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
* to deal in the Software without restriction, including without limitation
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice (including the next
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
* Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
* IN THE SOFTWARE.
|
|
*
|
|
*/
|
|
|
|
#include <linux/oom.h>
|
|
#include <linux/shmem_fs.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/swap.h>
|
|
#include <linux/pci.h>
|
|
#include <linux/dma-buf.h>
|
|
#include <linux/vmalloc.h>
|
|
#include <drm/drmP.h>
|
|
#include <drm/i915_drm.h>
|
|
|
|
#include "i915_drv.h"
|
|
#include "i915_trace.h"
|
|
|
|
static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
|
|
{
|
|
switch (mutex_trylock_recursive(&dev->struct_mutex)) {
|
|
case MUTEX_TRYLOCK_FAILED:
|
|
return false;
|
|
|
|
case MUTEX_TRYLOCK_SUCCESS:
|
|
*unlock = true;
|
|
return true;
|
|
|
|
case MUTEX_TRYLOCK_RECURSIVE:
|
|
*unlock = false;
|
|
return true;
|
|
}
|
|
|
|
BUG();
|
|
}
|
|
|
|
static bool any_vma_pinned(struct drm_i915_gem_object *obj)
|
|
{
|
|
struct i915_vma *vma;
|
|
|
|
list_for_each_entry(vma, &obj->vma_list, obj_link)
|
|
if (i915_vma_is_pinned(vma))
|
|
return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
static bool swap_available(void)
|
|
{
|
|
return get_nr_swap_pages() > 0;
|
|
}
|
|
|
|
static bool can_release_pages(struct drm_i915_gem_object *obj)
|
|
{
|
|
if (!obj->mm.pages)
|
|
return false;
|
|
|
|
/* Consider only shrinkable ojects. */
|
|
if (!i915_gem_object_is_shrinkable(obj))
|
|
return false;
|
|
|
|
/* Only report true if by unbinding the object and putting its pages
|
|
* we can actually make forward progress towards freeing physical
|
|
* pages.
|
|
*
|
|
* If the pages are pinned for any other reason than being bound
|
|
* to the GPU, simply unbinding from the GPU is not going to succeed
|
|
* in releasing our pin count on the pages themselves.
|
|
*/
|
|
if (atomic_read(&obj->mm.pages_pin_count) > obj->bind_count)
|
|
return false;
|
|
|
|
if (any_vma_pinned(obj))
|
|
return false;
|
|
|
|
/* We can only return physical pages to the system if we can either
|
|
* discard the contents (because the user has marked them as being
|
|
* purgeable) or if we can move their contents out to swap.
|
|
*/
|
|
return swap_available() || obj->mm.madv == I915_MADV_DONTNEED;
|
|
}
|
|
|
|
static bool unsafe_drop_pages(struct drm_i915_gem_object *obj)
|
|
{
|
|
if (i915_gem_object_unbind(obj) == 0)
|
|
__i915_gem_object_put_pages(obj, I915_MM_SHRINKER);
|
|
return !READ_ONCE(obj->mm.pages);
|
|
}
|
|
|
|
/**
|
|
* i915_gem_shrink - Shrink buffer object caches
|
|
* @dev_priv: i915 device
|
|
* @target: amount of memory to make available, in pages
|
|
* @flags: control flags for selecting cache types
|
|
*
|
|
* This function is the main interface to the shrinker. It will try to release
|
|
* up to @target pages of main memory backing storage from buffer objects.
|
|
* Selection of the specific caches can be done with @flags. This is e.g. useful
|
|
* when purgeable objects should be removed from caches preferentially.
|
|
*
|
|
* Note that it's not guaranteed that released amount is actually available as
|
|
* free system memory - the pages might still be in-used to due to other reasons
|
|
* (like cpu mmaps) or the mm core has reused them before we could grab them.
|
|
* Therefore code that needs to explicitly shrink buffer objects caches (e.g. to
|
|
* avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all().
|
|
*
|
|
* Also note that any kind of pinning (both per-vma address space pins and
|
|
* backing storage pins at the buffer object level) result in the shrinker code
|
|
* having to skip the object.
|
|
*
|
|
* Returns:
|
|
* The number of pages of backing storage actually released.
|
|
*/
|
|
unsigned long
|
|
i915_gem_shrink(struct drm_i915_private *dev_priv,
|
|
unsigned long target, unsigned flags)
|
|
{
|
|
const struct {
|
|
struct list_head *list;
|
|
unsigned int bit;
|
|
} phases[] = {
|
|
{ &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
|
|
{ &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
|
|
{ NULL, 0 },
|
|
}, *phase;
|
|
unsigned long count = 0;
|
|
bool unlock;
|
|
|
|
if (!i915_gem_shrinker_lock(&dev_priv->drm, &unlock))
|
|
return 0;
|
|
|
|
trace_i915_gem_shrink(dev_priv, target, flags);
|
|
i915_gem_retire_requests(dev_priv);
|
|
|
|
/*
|
|
* Unbinding of objects will require HW access; Let us not wake the
|
|
* device just to recover a little memory. If absolutely necessary,
|
|
* we will force the wake during oom-notifier.
|
|
*/
|
|
if ((flags & I915_SHRINK_BOUND) &&
|
|
!intel_runtime_pm_get_if_in_use(dev_priv))
|
|
flags &= ~I915_SHRINK_BOUND;
|
|
|
|
/*
|
|
* As we may completely rewrite the (un)bound list whilst unbinding
|
|
* (due to retiring requests) we have to strictly process only
|
|
* one element of the list at the time, and recheck the list
|
|
* on every iteration.
|
|
*
|
|
* In particular, we must hold a reference whilst removing the
|
|
* object as we may end up waiting for and/or retiring the objects.
|
|
* This might release the final reference (held by the active list)
|
|
* and result in the object being freed from under us. This is
|
|
* similar to the precautions the eviction code must take whilst
|
|
* removing objects.
|
|
*
|
|
* Also note that although these lists do not hold a reference to
|
|
* the object we can safely grab one here: The final object
|
|
* unreferencing and the bound_list are both protected by the
|
|
* dev->struct_mutex and so we won't ever be able to observe an
|
|
* object on the bound_list with a reference count equals 0.
|
|
*/
|
|
for (phase = phases; phase->list; phase++) {
|
|
struct list_head still_in_list;
|
|
struct drm_i915_gem_object *obj;
|
|
|
|
if ((flags & phase->bit) == 0)
|
|
continue;
|
|
|
|
INIT_LIST_HEAD(&still_in_list);
|
|
while (count < target &&
|
|
(obj = list_first_entry_or_null(phase->list,
|
|
typeof(*obj),
|
|
global_link))) {
|
|
list_move_tail(&obj->global_link, &still_in_list);
|
|
if (!obj->mm.pages) {
|
|
list_del_init(&obj->global_link);
|
|
continue;
|
|
}
|
|
|
|
if (flags & I915_SHRINK_PURGEABLE &&
|
|
obj->mm.madv != I915_MADV_DONTNEED)
|
|
continue;
|
|
|
|
if (flags & I915_SHRINK_VMAPS &&
|
|
!is_vmalloc_addr(obj->mm.mapping))
|
|
continue;
|
|
|
|
if (!(flags & I915_SHRINK_ACTIVE) &&
|
|
(i915_gem_object_is_active(obj) ||
|
|
i915_gem_object_is_framebuffer(obj)))
|
|
continue;
|
|
|
|
if (!can_release_pages(obj))
|
|
continue;
|
|
|
|
if (unsafe_drop_pages(obj)) {
|
|
/* May arrive from get_pages on another bo */
|
|
mutex_lock_nested(&obj->mm.lock,
|
|
I915_MM_SHRINKER);
|
|
if (!obj->mm.pages) {
|
|
__i915_gem_object_invalidate(obj);
|
|
list_del_init(&obj->global_link);
|
|
count += obj->base.size >> PAGE_SHIFT;
|
|
}
|
|
mutex_unlock(&obj->mm.lock);
|
|
}
|
|
}
|
|
list_splice_tail(&still_in_list, phase->list);
|
|
}
|
|
|
|
if (flags & I915_SHRINK_BOUND)
|
|
intel_runtime_pm_put(dev_priv);
|
|
|
|
i915_gem_retire_requests(dev_priv);
|
|
if (unlock)
|
|
mutex_unlock(&dev_priv->drm.struct_mutex);
|
|
|
|
/* expedite the RCU grace period to free some request slabs */
|
|
synchronize_rcu_expedited();
|
|
|
|
return count;
|
|
}
|
|
|
|
/**
|
|
* i915_gem_shrink_all - Shrink buffer object caches completely
|
|
* @dev_priv: i915 device
|
|
*
|
|
* This is a simple wraper around i915_gem_shrink() to aggressively shrink all
|
|
* caches completely. It also first waits for and retires all outstanding
|
|
* requests to also be able to release backing storage for active objects.
|
|
*
|
|
* This should only be used in code to intentionally quiescent the gpu or as a
|
|
* last-ditch effort when memory seems to have run out.
|
|
*
|
|
* Returns:
|
|
* The number of pages of backing storage actually released.
|
|
*/
|
|
unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
|
|
{
|
|
unsigned long freed;
|
|
|
|
intel_runtime_pm_get(dev_priv);
|
|
freed = i915_gem_shrink(dev_priv, -1UL,
|
|
I915_SHRINK_BOUND |
|
|
I915_SHRINK_UNBOUND |
|
|
I915_SHRINK_ACTIVE);
|
|
intel_runtime_pm_put(dev_priv);
|
|
|
|
synchronize_rcu(); /* wait for our earlier RCU delayed slab frees */
|
|
|
|
return freed;
|
|
}
|
|
|
|
static unsigned long
|
|
i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
|
|
{
|
|
struct drm_i915_private *dev_priv =
|
|
container_of(shrinker, struct drm_i915_private, mm.shrinker);
|
|
struct drm_device *dev = &dev_priv->drm;
|
|
struct drm_i915_gem_object *obj;
|
|
unsigned long count;
|
|
bool unlock;
|
|
|
|
if (!i915_gem_shrinker_lock(dev, &unlock))
|
|
return 0;
|
|
|
|
i915_gem_retire_requests(dev_priv);
|
|
|
|
count = 0;
|
|
list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link)
|
|
if (can_release_pages(obj))
|
|
count += obj->base.size >> PAGE_SHIFT;
|
|
|
|
list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) {
|
|
if (!i915_gem_object_is_active(obj) && can_release_pages(obj))
|
|
count += obj->base.size >> PAGE_SHIFT;
|
|
}
|
|
|
|
if (unlock)
|
|
mutex_unlock(&dev->struct_mutex);
|
|
|
|
return count;
|
|
}
|
|
|
|
static unsigned long
|
|
i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
|
|
{
|
|
struct drm_i915_private *dev_priv =
|
|
container_of(shrinker, struct drm_i915_private, mm.shrinker);
|
|
struct drm_device *dev = &dev_priv->drm;
|
|
unsigned long freed;
|
|
bool unlock;
|
|
|
|
if (!i915_gem_shrinker_lock(dev, &unlock))
|
|
return SHRINK_STOP;
|
|
|
|
freed = i915_gem_shrink(dev_priv,
|
|
sc->nr_to_scan,
|
|
I915_SHRINK_BOUND |
|
|
I915_SHRINK_UNBOUND |
|
|
I915_SHRINK_PURGEABLE);
|
|
if (freed < sc->nr_to_scan)
|
|
freed += i915_gem_shrink(dev_priv,
|
|
sc->nr_to_scan - freed,
|
|
I915_SHRINK_BOUND |
|
|
I915_SHRINK_UNBOUND);
|
|
if (unlock)
|
|
mutex_unlock(&dev->struct_mutex);
|
|
|
|
return freed;
|
|
}
|
|
|
|
struct shrinker_lock_uninterruptible {
|
|
bool was_interruptible;
|
|
bool unlock;
|
|
};
|
|
|
|
static bool
|
|
i915_gem_shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv,
|
|
struct shrinker_lock_uninterruptible *slu,
|
|
int timeout_ms)
|
|
{
|
|
unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms);
|
|
|
|
do {
|
|
if (i915_gem_wait_for_idle(dev_priv, 0) == 0 &&
|
|
i915_gem_shrinker_lock(&dev_priv->drm, &slu->unlock))
|
|
break;
|
|
|
|
schedule_timeout_killable(1);
|
|
if (fatal_signal_pending(current))
|
|
return false;
|
|
|
|
if (time_after(jiffies, timeout)) {
|
|
pr_err("Unable to lock GPU to purge memory.\n");
|
|
return false;
|
|
}
|
|
} while (1);
|
|
|
|
slu->was_interruptible = dev_priv->mm.interruptible;
|
|
dev_priv->mm.interruptible = false;
|
|
return true;
|
|
}
|
|
|
|
static void
|
|
i915_gem_shrinker_unlock_uninterruptible(struct drm_i915_private *dev_priv,
|
|
struct shrinker_lock_uninterruptible *slu)
|
|
{
|
|
dev_priv->mm.interruptible = slu->was_interruptible;
|
|
if (slu->unlock)
|
|
mutex_unlock(&dev_priv->drm.struct_mutex);
|
|
}
|
|
|
|
static int
|
|
i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
|
|
{
|
|
struct drm_i915_private *dev_priv =
|
|
container_of(nb, struct drm_i915_private, mm.oom_notifier);
|
|
struct shrinker_lock_uninterruptible slu;
|
|
struct drm_i915_gem_object *obj;
|
|
unsigned long unevictable, bound, unbound, freed_pages;
|
|
|
|
if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000))
|
|
return NOTIFY_DONE;
|
|
|
|
freed_pages = i915_gem_shrink_all(dev_priv);
|
|
|
|
/* Because we may be allocating inside our own driver, we cannot
|
|
* assert that there are no objects with pinned pages that are not
|
|
* being pointed to by hardware.
|
|
*/
|
|
unbound = bound = unevictable = 0;
|
|
list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) {
|
|
if (!obj->mm.pages)
|
|
continue;
|
|
|
|
if (!can_release_pages(obj))
|
|
unevictable += obj->base.size >> PAGE_SHIFT;
|
|
else
|
|
unbound += obj->base.size >> PAGE_SHIFT;
|
|
}
|
|
list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) {
|
|
if (!obj->mm.pages)
|
|
continue;
|
|
|
|
if (!can_release_pages(obj))
|
|
unevictable += obj->base.size >> PAGE_SHIFT;
|
|
else
|
|
bound += obj->base.size >> PAGE_SHIFT;
|
|
}
|
|
|
|
i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu);
|
|
|
|
if (freed_pages || unbound || bound)
|
|
pr_info("Purging GPU memory, %lu pages freed, "
|
|
"%lu pages still pinned.\n",
|
|
freed_pages, unevictable);
|
|
if (unbound || bound)
|
|
pr_err("%lu and %lu pages still available in the "
|
|
"bound and unbound GPU page lists.\n",
|
|
bound, unbound);
|
|
|
|
*(unsigned long *)ptr += freed_pages;
|
|
return NOTIFY_DONE;
|
|
}
|
|
|
|
static int
|
|
i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
|
|
{
|
|
struct drm_i915_private *dev_priv =
|
|
container_of(nb, struct drm_i915_private, mm.vmap_notifier);
|
|
struct shrinker_lock_uninterruptible slu;
|
|
struct i915_vma *vma, *next;
|
|
unsigned long freed_pages = 0;
|
|
int ret;
|
|
|
|
if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000))
|
|
return NOTIFY_DONE;
|
|
|
|
/* Force everything onto the inactive lists */
|
|
ret = i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED);
|
|
if (ret)
|
|
goto out;
|
|
|
|
intel_runtime_pm_get(dev_priv);
|
|
freed_pages += i915_gem_shrink(dev_priv, -1UL,
|
|
I915_SHRINK_BOUND |
|
|
I915_SHRINK_UNBOUND |
|
|
I915_SHRINK_ACTIVE |
|
|
I915_SHRINK_VMAPS);
|
|
intel_runtime_pm_put(dev_priv);
|
|
|
|
/* We also want to clear any cached iomaps as they wrap vmap */
|
|
list_for_each_entry_safe(vma, next,
|
|
&dev_priv->ggtt.base.inactive_list, vm_link) {
|
|
unsigned long count = vma->node.size >> PAGE_SHIFT;
|
|
if (vma->iomap && i915_vma_unbind(vma) == 0)
|
|
freed_pages += count;
|
|
}
|
|
|
|
out:
|
|
i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu);
|
|
|
|
*(unsigned long *)ptr += freed_pages;
|
|
return NOTIFY_DONE;
|
|
}
|
|
|
|
/**
|
|
* i915_gem_shrinker_init - Initialize i915 shrinker
|
|
* @dev_priv: i915 device
|
|
*
|
|
* This function registers and sets up the i915 shrinker and OOM handler.
|
|
*/
|
|
void i915_gem_shrinker_init(struct drm_i915_private *dev_priv)
|
|
{
|
|
dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
|
|
dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count;
|
|
dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS;
|
|
WARN_ON(register_shrinker(&dev_priv->mm.shrinker));
|
|
|
|
dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
|
|
WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier));
|
|
|
|
dev_priv->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap;
|
|
WARN_ON(register_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
|
|
}
|
|
|
|
/**
|
|
* i915_gem_shrinker_cleanup - Clean up i915 shrinker
|
|
* @dev_priv: i915 device
|
|
*
|
|
* This function unregisters the i915 shrinker and OOM handler.
|
|
*/
|
|
void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv)
|
|
{
|
|
WARN_ON(unregister_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
|
|
WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier));
|
|
unregister_shrinker(&dev_priv->mm.shrinker);
|
|
}
|