diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c index f8827c23a668..dddca007912a 100644 --- a/drivers/gpu/drm/i915/i915_dma.c +++ b/drivers/gpu/drm/i915/i915_dma.c @@ -2096,6 +2096,9 @@ int i915_driver_unload(struct drm_device *dev) i915_mch_dev = NULL; spin_unlock(&mchdev_lock); + if (dev_priv->mm.inactive_shrinker.shrink) + unregister_shrinker(&dev_priv->mm.inactive_shrinker); + mutex_lock(&dev->struct_mutex); ret = i915_gpu_idle(dev); if (ret) diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 82c19ab3e1e2..d22aab94d645 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -660,8 +660,6 @@ static int __init i915_init(void) driver.num_ioctls = i915_max_ioctl; - i915_gem_shrinker_init(); - /* * If CONFIG_DRM_I915_KMS is set, default to KMS unless * explicitly disabled with the module pararmeter. @@ -693,7 +691,6 @@ static int __init i915_init(void) static void __exit i915_exit(void) { - i915_gem_shrinker_exit(); drm_exit(&driver); } diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 895f9173de58..29e8b7962edf 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -542,14 +542,7 @@ typedef struct drm_i915_private { struct io_mapping *gtt_mapping; int gtt_mtrr; - /** - * Membership on list of all loaded devices, used to evict - * inactive buffers under memory pressure. - * - * Modifications should only be done whilst holding the - * shrink_list_lock spinlock. - */ - struct list_head shrink_list; + struct shrinker inactive_shrinker; /** * List of objects currently involved in rendering. @@ -1079,9 +1072,6 @@ void i915_gem_detach_phys_object(struct drm_device *dev, void i915_gem_free_all_phys_object(struct drm_device *dev); void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv); -void i915_gem_shrinker_init(void); -void i915_gem_shrinker_exit(void); - /* i915_gem_evict.c */ int i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment, bool mappable); diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 19ceb8cd0922..1af7285ca162 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -65,8 +65,10 @@ i915_gem_object_get_pages(struct drm_gem_object *obj, static void i915_gem_object_put_pages(struct drm_gem_object *obj); -static LIST_HEAD(shrink_list); -static DEFINE_SPINLOCK(shrink_list_lock); +static int i915_gem_inactive_shrink(struct shrinker *shrinker, + int nr_to_scan, + gfp_t gfp_mask); + /* some bookkeeping */ static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, @@ -4765,9 +4767,6 @@ i915_gem_load(struct drm_device *dev) INIT_DELAYED_WORK(&dev_priv->mm.retire_work, i915_gem_retire_work_handler); init_completion(&dev_priv->error_completion); - spin_lock(&shrink_list_lock); - list_add(&dev_priv->mm.shrink_list, &shrink_list); - spin_unlock(&shrink_list_lock); /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ if (IS_GEN3(dev)) { @@ -4810,6 +4809,10 @@ i915_gem_load(struct drm_device *dev) } i915_gem_detect_bit_6_swizzle(dev); init_waitqueue_head(&dev_priv->pending_flip_queue); + + dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink; + dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS; + register_shrinker(&dev_priv->mm.inactive_shrinker); } /* @@ -5022,152 +5025,74 @@ i915_gpu_is_active(struct drm_device *dev) int lists_empty; lists_empty = list_empty(&dev_priv->mm.flushing_list) && - list_empty(&dev_priv->render_ring.active_list) && - list_empty(&dev_priv->bsd_ring.active_list) && - list_empty(&dev_priv->blt_ring.active_list); + list_empty(&dev_priv->mm.active_list); return !lists_empty; } static int -i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask) +i915_gem_inactive_shrink(struct shrinker *shrinker, + int nr_to_scan, + gfp_t gfp_mask) { - drm_i915_private_t *dev_priv, *next_dev; - struct drm_i915_gem_object *obj_priv, *next_obj; - int cnt = 0; - int would_deadlock = 1; + struct drm_i915_private *dev_priv = + container_of(shrinker, + struct drm_i915_private, + mm.inactive_shrinker); + struct drm_device *dev = dev_priv->dev; + struct drm_i915_gem_object *obj, *next; + int cnt; + + if (!mutex_trylock(&dev->struct_mutex)) + return nr_to_scan ? 0 : -1; /* "fast-path" to count number of available objects */ if (nr_to_scan == 0) { - spin_lock(&shrink_list_lock); - list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) { - struct drm_device *dev = dev_priv->dev; - - if (mutex_trylock(&dev->struct_mutex)) { - list_for_each_entry(obj_priv, - &dev_priv->mm.inactive_list, - mm_list) - cnt++; - mutex_unlock(&dev->struct_mutex); - } - } - spin_unlock(&shrink_list_lock); - - return (cnt / 100) * sysctl_vfs_cache_pressure; + cnt = 0; + list_for_each_entry(obj, + &dev_priv->mm.inactive_list, + mm_list) + cnt++; + mutex_unlock(&dev->struct_mutex); + return cnt / 100 * sysctl_vfs_cache_pressure; } - spin_lock(&shrink_list_lock); - rescan: /* first scan for clean buffers */ - list_for_each_entry_safe(dev_priv, next_dev, - &shrink_list, mm.shrink_list) { - struct drm_device *dev = dev_priv->dev; + i915_gem_retire_requests(dev); - if (! mutex_trylock(&dev->struct_mutex)) - continue; - - spin_unlock(&shrink_list_lock); - i915_gem_retire_requests(dev); - - list_for_each_entry_safe(obj_priv, next_obj, - &dev_priv->mm.inactive_list, - mm_list) { - if (i915_gem_object_is_purgeable(obj_priv)) { - i915_gem_object_unbind(&obj_priv->base); - if (--nr_to_scan <= 0) - break; - } + list_for_each_entry_safe(obj, next, + &dev_priv->mm.inactive_list, + mm_list) { + if (i915_gem_object_is_purgeable(obj)) { + i915_gem_object_unbind(&obj->base); + if (--nr_to_scan == 0) + break; } - - spin_lock(&shrink_list_lock); - mutex_unlock(&dev->struct_mutex); - - would_deadlock = 0; - - if (nr_to_scan <= 0) - break; } /* second pass, evict/count anything still on the inactive list */ - list_for_each_entry_safe(dev_priv, next_dev, - &shrink_list, mm.shrink_list) { - struct drm_device *dev = dev_priv->dev; - - if (! mutex_trylock(&dev->struct_mutex)) - continue; - - spin_unlock(&shrink_list_lock); - - list_for_each_entry_safe(obj_priv, next_obj, - &dev_priv->mm.inactive_list, - mm_list) { - if (nr_to_scan > 0) { - i915_gem_object_unbind(&obj_priv->base); - nr_to_scan--; - } else - cnt++; - } - - spin_lock(&shrink_list_lock); - mutex_unlock(&dev->struct_mutex); - - would_deadlock = 0; + cnt = 0; + list_for_each_entry_safe(obj, next, + &dev_priv->mm.inactive_list, + mm_list) { + if (nr_to_scan) { + i915_gem_object_unbind(&obj->base); + nr_to_scan--; + } else + cnt++; } - if (nr_to_scan) { - int active = 0; - + if (nr_to_scan && i915_gpu_is_active(dev)) { /* * We are desperate for pages, so as a last resort, wait * for the GPU to finish and discard whatever we can. * This has a dramatic impact to reduce the number of * OOM-killer events whilst running the GPU aggressively. */ - list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) { - struct drm_device *dev = dev_priv->dev; - - if (!mutex_trylock(&dev->struct_mutex)) - continue; - - spin_unlock(&shrink_list_lock); - - if (i915_gpu_is_active(dev)) { - i915_gpu_idle(dev); - active++; - } - - spin_lock(&shrink_list_lock); - mutex_unlock(&dev->struct_mutex); - } - - if (active) + if (i915_gpu_idle(dev) == 0) goto rescan; } - - spin_unlock(&shrink_list_lock); - - if (would_deadlock) - return -1; - else if (cnt > 0) - return (cnt / 100) * sysctl_vfs_cache_pressure; - else - return 0; -} - -static struct shrinker shrinker = { - .shrink = i915_gem_shrink, - .seeks = DEFAULT_SEEKS, -}; - -__init void -i915_gem_shrinker_init(void) -{ - register_shrinker(&shrinker); -} - -__exit void -i915_gem_shrinker_exit(void) -{ - unregister_shrinker(&shrinker); + mutex_unlock(&dev->struct_mutex); + return cnt / 100 * sysctl_vfs_cache_pressure; }