diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c index 2ac074c8f5d2..da99edc50888 100644 --- a/drivers/gpu/drm/drm_mm.c +++ b/drivers/gpu/drm/drm_mm.c @@ -48,44 +48,14 @@ #define MM_UNUSED_TARGET 4 -unsigned long drm_mm_tail_space(struct drm_mm *mm) -{ - struct list_head *tail_node; - struct drm_mm_node *entry; - - tail_node = mm->ml_entry.prev; - entry = list_entry(tail_node, struct drm_mm_node, ml_entry); - if (!entry->free) - return 0; - - return entry->size; -} - -int drm_mm_remove_space_from_tail(struct drm_mm *mm, unsigned long size) -{ - struct list_head *tail_node; - struct drm_mm_node *entry; - - tail_node = mm->ml_entry.prev; - entry = list_entry(tail_node, struct drm_mm_node, ml_entry); - if (!entry->free) - return -ENOMEM; - - if (entry->size <= size) - return -ENOMEM; - - entry->size -= size; - return 0; -} - static struct drm_mm_node *drm_mm_kmalloc(struct drm_mm *mm, int atomic) { struct drm_mm_node *child; if (atomic) - child = kmalloc(sizeof(*child), GFP_ATOMIC); + child = kzalloc(sizeof(*child), GFP_ATOMIC); else - child = kmalloc(sizeof(*child), GFP_KERNEL); + child = kzalloc(sizeof(*child), GFP_KERNEL); if (unlikely(child == NULL)) { spin_lock(&mm->unused_lock); @@ -94,8 +64,8 @@ static struct drm_mm_node *drm_mm_kmalloc(struct drm_mm *mm, int atomic) else { child = list_entry(mm->unused_nodes.next, - struct drm_mm_node, fl_entry); - list_del(&child->fl_entry); + struct drm_mm_node, free_stack); + list_del(&child->free_stack); --mm->num_unused; } spin_unlock(&mm->unused_lock); @@ -115,7 +85,7 @@ int drm_mm_pre_get(struct drm_mm *mm) spin_lock(&mm->unused_lock); while (mm->num_unused < MM_UNUSED_TARGET) { spin_unlock(&mm->unused_lock); - node = kmalloc(sizeof(*node), GFP_KERNEL); + node = kzalloc(sizeof(*node), GFP_KERNEL); spin_lock(&mm->unused_lock); if (unlikely(node == NULL)) { @@ -124,7 +94,7 @@ int drm_mm_pre_get(struct drm_mm *mm) return ret; } ++mm->num_unused; - list_add_tail(&node->fl_entry, &mm->unused_nodes); + list_add_tail(&node->free_stack, &mm->unused_nodes); } spin_unlock(&mm->unused_lock); return 0; @@ -146,27 +116,12 @@ static int drm_mm_create_tail_node(struct drm_mm *mm, child->start = start; child->mm = mm; - list_add_tail(&child->ml_entry, &mm->ml_entry); - list_add_tail(&child->fl_entry, &mm->fl_entry); + list_add_tail(&child->node_list, &mm->node_list); + list_add_tail(&child->free_stack, &mm->free_stack); return 0; } -int drm_mm_add_space_to_tail(struct drm_mm *mm, unsigned long size, int atomic) -{ - struct list_head *tail_node; - struct drm_mm_node *entry; - - tail_node = mm->ml_entry.prev; - entry = list_entry(tail_node, struct drm_mm_node, ml_entry); - if (!entry->free) { - return drm_mm_create_tail_node(mm, entry->start + entry->size, - size, atomic); - } - entry->size += size; - return 0; -} - static struct drm_mm_node *drm_mm_split_at_start(struct drm_mm_node *parent, unsigned long size, int atomic) @@ -177,15 +132,14 @@ static struct drm_mm_node *drm_mm_split_at_start(struct drm_mm_node *parent, if (unlikely(child == NULL)) return NULL; - INIT_LIST_HEAD(&child->fl_entry); + INIT_LIST_HEAD(&child->free_stack); - child->free = 0; child->size = size; child->start = parent->start; child->mm = parent->mm; - list_add_tail(&child->ml_entry, &parent->ml_entry); - INIT_LIST_HEAD(&child->fl_entry); + list_add_tail(&child->node_list, &parent->node_list); + INIT_LIST_HEAD(&child->free_stack); parent->size -= size; parent->start += size; @@ -213,7 +167,7 @@ struct drm_mm_node *drm_mm_get_block_generic(struct drm_mm_node *node, } if (node->size == size) { - list_del_init(&node->fl_entry); + list_del_init(&node->free_stack); node->free = 0; } else { node = drm_mm_split_at_start(node, size, atomic); @@ -251,7 +205,7 @@ struct drm_mm_node *drm_mm_get_block_range_generic(struct drm_mm_node *node, } if (node->size == size) { - list_del_init(&node->fl_entry); + list_del_init(&node->free_stack); node->free = 0; } else { node = drm_mm_split_at_start(node, size, atomic); @@ -273,16 +227,19 @@ void drm_mm_put_block(struct drm_mm_node *cur) { struct drm_mm *mm = cur->mm; - struct list_head *cur_head = &cur->ml_entry; - struct list_head *root_head = &mm->ml_entry; + struct list_head *cur_head = &cur->node_list; + struct list_head *root_head = &mm->node_list; struct drm_mm_node *prev_node = NULL; struct drm_mm_node *next_node; int merged = 0; + BUG_ON(cur->scanned_block || cur->scanned_prev_free + || cur->scanned_next_free); + if (cur_head->prev != root_head) { prev_node = - list_entry(cur_head->prev, struct drm_mm_node, ml_entry); + list_entry(cur_head->prev, struct drm_mm_node, node_list); if (prev_node->free) { prev_node->size += cur->size; merged = 1; @@ -290,15 +247,15 @@ void drm_mm_put_block(struct drm_mm_node *cur) } if (cur_head->next != root_head) { next_node = - list_entry(cur_head->next, struct drm_mm_node, ml_entry); + list_entry(cur_head->next, struct drm_mm_node, node_list); if (next_node->free) { if (merged) { prev_node->size += next_node->size; - list_del(&next_node->ml_entry); - list_del(&next_node->fl_entry); + list_del(&next_node->node_list); + list_del(&next_node->free_stack); spin_lock(&mm->unused_lock); if (mm->num_unused < MM_UNUSED_TARGET) { - list_add(&next_node->fl_entry, + list_add(&next_node->free_stack, &mm->unused_nodes); ++mm->num_unused; } else @@ -313,12 +270,12 @@ void drm_mm_put_block(struct drm_mm_node *cur) } if (!merged) { cur->free = 1; - list_add(&cur->fl_entry, &mm->fl_entry); + list_add(&cur->free_stack, &mm->free_stack); } else { - list_del(&cur->ml_entry); + list_del(&cur->node_list); spin_lock(&mm->unused_lock); if (mm->num_unused < MM_UNUSED_TARGET) { - list_add(&cur->fl_entry, &mm->unused_nodes); + list_add(&cur->free_stack, &mm->unused_nodes); ++mm->num_unused; } else kfree(cur); @@ -328,40 +285,50 @@ void drm_mm_put_block(struct drm_mm_node *cur) EXPORT_SYMBOL(drm_mm_put_block); +static int check_free_mm_node(struct drm_mm_node *entry, unsigned long size, + unsigned alignment) +{ + unsigned wasted = 0; + + if (entry->size < size) + return 0; + + if (alignment) { + register unsigned tmp = entry->start % alignment; + if (tmp) + wasted = alignment - tmp; + } + + if (entry->size >= size + wasted) { + return 1; + } + + return 0; +} + struct drm_mm_node *drm_mm_search_free(const struct drm_mm *mm, unsigned long size, unsigned alignment, int best_match) { - struct list_head *list; - const struct list_head *free_stack = &mm->fl_entry; struct drm_mm_node *entry; struct drm_mm_node *best; unsigned long best_size; - unsigned wasted; + + BUG_ON(mm->scanned_blocks); best = NULL; best_size = ~0UL; - list_for_each(list, free_stack) { - entry = list_entry(list, struct drm_mm_node, fl_entry); - wasted = 0; - - if (entry->size < size) + list_for_each_entry(entry, &mm->free_stack, free_stack) { + if (!check_free_mm_node(entry, size, alignment)) continue; - if (alignment) { - register unsigned tmp = entry->start % alignment; - if (tmp) - wasted += alignment - tmp; - } + if (!best_match) + return entry; - if (entry->size >= size + wasted) { - if (!best_match) - return entry; - if (entry->size < best_size) { - best = entry; - best_size = entry->size; - } + if (entry->size < best_size) { + best = entry; + best_size = entry->size; } } @@ -376,43 +343,28 @@ struct drm_mm_node *drm_mm_search_free_in_range(const struct drm_mm *mm, unsigned long end, int best_match) { - struct list_head *list; - const struct list_head *free_stack = &mm->fl_entry; struct drm_mm_node *entry; struct drm_mm_node *best; unsigned long best_size; - unsigned wasted; + + BUG_ON(mm->scanned_blocks); best = NULL; best_size = ~0UL; - list_for_each(list, free_stack) { - entry = list_entry(list, struct drm_mm_node, fl_entry); - wasted = 0; - - if (entry->size < size) - continue; - + list_for_each_entry(entry, &mm->free_stack, free_stack) { if (entry->start > end || (entry->start+entry->size) < start) continue; - if (entry->start < start) - wasted += start - entry->start; + if (!check_free_mm_node(entry, size, alignment)) + continue; - if (alignment) { - register unsigned tmp = (entry->start + wasted) % alignment; - if (tmp) - wasted += alignment - tmp; - } + if (!best_match) + return entry; - if (entry->size >= size + wasted && - (entry->start + wasted + size) <= end) { - if (!best_match) - return entry; - if (entry->size < best_size) { - best = entry; - best_size = entry->size; - } + if (entry->size < best_size) { + best = entry; + best_size = entry->size; } } @@ -420,9 +372,161 @@ struct drm_mm_node *drm_mm_search_free_in_range(const struct drm_mm *mm, } EXPORT_SYMBOL(drm_mm_search_free_in_range); +/** + * Initializa lru scanning. + * + * This simply sets up the scanning routines with the parameters for the desired + * hole. + * + * Warning: As long as the scan list is non-empty, no other operations than + * adding/removing nodes to/from the scan list are allowed. + */ +void drm_mm_init_scan(struct drm_mm *mm, unsigned long size, + unsigned alignment) +{ + mm->scan_alignment = alignment; + mm->scan_size = size; + mm->scanned_blocks = 0; + mm->scan_hit_start = 0; + mm->scan_hit_size = 0; +} +EXPORT_SYMBOL(drm_mm_init_scan); + +/** + * Add a node to the scan list that might be freed to make space for the desired + * hole. + * + * Returns non-zero, if a hole has been found, zero otherwise. + */ +int drm_mm_scan_add_block(struct drm_mm_node *node) +{ + struct drm_mm *mm = node->mm; + struct list_head *prev_free, *next_free; + struct drm_mm_node *prev_node, *next_node; + + mm->scanned_blocks++; + + prev_free = next_free = NULL; + + BUG_ON(node->free); + node->scanned_block = 1; + node->free = 1; + + if (node->node_list.prev != &mm->node_list) { + prev_node = list_entry(node->node_list.prev, struct drm_mm_node, + node_list); + + if (prev_node->free) { + list_del(&prev_node->node_list); + + node->start = prev_node->start; + node->size += prev_node->size; + + prev_node->scanned_prev_free = 1; + + prev_free = &prev_node->free_stack; + } + } + + if (node->node_list.next != &mm->node_list) { + next_node = list_entry(node->node_list.next, struct drm_mm_node, + node_list); + + if (next_node->free) { + list_del(&next_node->node_list); + + node->size += next_node->size; + + next_node->scanned_next_free = 1; + + next_free = &next_node->free_stack; + } + } + + /* The free_stack list is not used for allocated objects, so these two + * pointers can be abused (as long as no allocations in this memory + * manager happens). */ + node->free_stack.prev = prev_free; + node->free_stack.next = next_free; + + if (check_free_mm_node(node, mm->scan_size, mm->scan_alignment)) { + mm->scan_hit_start = node->start; + mm->scan_hit_size = node->size; + + return 1; + } + + return 0; +} +EXPORT_SYMBOL(drm_mm_scan_add_block); + +/** + * Remove a node from the scan list. + * + * Nodes _must_ be removed in the exact same order from the scan list as they + * have been added, otherwise the internal state of the memory manager will be + * corrupted. + * + * When the scan list is empty, the selected memory nodes can be freed. An + * immediatly following drm_mm_search_free with best_match = 0 will then return + * the just freed block (because its at the top of the free_stack list). + * + * Returns one if this block should be evicted, zero otherwise. Will always + * return zero when no hole has been found. + */ +int drm_mm_scan_remove_block(struct drm_mm_node *node) +{ + struct drm_mm *mm = node->mm; + struct drm_mm_node *prev_node, *next_node; + + mm->scanned_blocks--; + + BUG_ON(!node->scanned_block); + node->scanned_block = 0; + node->free = 0; + + prev_node = list_entry(node->free_stack.prev, struct drm_mm_node, + free_stack); + next_node = list_entry(node->free_stack.next, struct drm_mm_node, + free_stack); + + if (prev_node) { + BUG_ON(!prev_node->scanned_prev_free); + prev_node->scanned_prev_free = 0; + + list_add_tail(&prev_node->node_list, &node->node_list); + + node->start = prev_node->start + prev_node->size; + node->size -= prev_node->size; + } + + if (next_node) { + BUG_ON(!next_node->scanned_next_free); + next_node->scanned_next_free = 0; + + list_add(&next_node->node_list, &node->node_list); + + node->size -= next_node->size; + } + + INIT_LIST_HEAD(&node->free_stack); + + /* Only need to check for containement because start&size for the + * complete resulting free block (not just the desired part) is + * stored. */ + if (node->start >= mm->scan_hit_start && + node->start + node->size + <= mm->scan_hit_start + mm->scan_hit_size) { + return 1; + } + + return 0; +} +EXPORT_SYMBOL(drm_mm_scan_remove_block); + int drm_mm_clean(struct drm_mm * mm) { - struct list_head *head = &mm->ml_entry; + struct list_head *head = &mm->node_list; return (head->next->next == head); } @@ -430,10 +534,11 @@ EXPORT_SYMBOL(drm_mm_clean); int drm_mm_init(struct drm_mm * mm, unsigned long start, unsigned long size) { - INIT_LIST_HEAD(&mm->ml_entry); - INIT_LIST_HEAD(&mm->fl_entry); + INIT_LIST_HEAD(&mm->node_list); + INIT_LIST_HEAD(&mm->free_stack); INIT_LIST_HEAD(&mm->unused_nodes); mm->num_unused = 0; + mm->scanned_blocks = 0; spin_lock_init(&mm->unused_lock); return drm_mm_create_tail_node(mm, start, size, 0); @@ -442,25 +547,25 @@ EXPORT_SYMBOL(drm_mm_init); void drm_mm_takedown(struct drm_mm * mm) { - struct list_head *bnode = mm->fl_entry.next; + struct list_head *bnode = mm->free_stack.next; struct drm_mm_node *entry; struct drm_mm_node *next; - entry = list_entry(bnode, struct drm_mm_node, fl_entry); + entry = list_entry(bnode, struct drm_mm_node, free_stack); - if (entry->ml_entry.next != &mm->ml_entry || - entry->fl_entry.next != &mm->fl_entry) { + if (entry->node_list.next != &mm->node_list || + entry->free_stack.next != &mm->free_stack) { DRM_ERROR("Memory manager not clean. Delaying takedown\n"); return; } - list_del(&entry->fl_entry); - list_del(&entry->ml_entry); + list_del(&entry->free_stack); + list_del(&entry->node_list); kfree(entry); spin_lock(&mm->unused_lock); - list_for_each_entry_safe(entry, next, &mm->unused_nodes, fl_entry) { - list_del(&entry->fl_entry); + list_for_each_entry_safe(entry, next, &mm->unused_nodes, free_stack) { + list_del(&entry->free_stack); kfree(entry); --mm->num_unused; } @@ -475,7 +580,7 @@ void drm_mm_debug_table(struct drm_mm *mm, const char *prefix) struct drm_mm_node *entry; int total_used = 0, total_free = 0, total = 0; - list_for_each_entry(entry, &mm->ml_entry, ml_entry) { + list_for_each_entry(entry, &mm->node_list, node_list) { printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8ld: %s\n", prefix, entry->start, entry->start + entry->size, entry->size, entry->free ? "free" : "used"); @@ -496,7 +601,7 @@ int drm_mm_dump_table(struct seq_file *m, struct drm_mm *mm) struct drm_mm_node *entry; int total_used = 0, total_free = 0, total = 0; - list_for_each_entry(entry, &mm->ml_entry, ml_entry) { + list_for_each_entry(entry, &mm->node_list, node_list) { seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: %s\n", entry->start, entry->start + entry->size, entry->size, entry->free ? "free" : "used"); total += entry->size; if (entry->free) diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 074385882ccf..75061b305b8c 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -2633,10 +2633,8 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) if (free_space != NULL) { obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size, alignment); - if (obj_priv->gtt_space != NULL) { - obj_priv->gtt_space->private = obj; + if (obj_priv->gtt_space != NULL) obj_priv->gtt_offset = obj_priv->gtt_space->start; - } } if (obj_priv->gtt_space == NULL) { /* If the gtt is empty and we're still having trouble diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 555ebb12ace8..9763288c6b2d 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -476,7 +476,6 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool remove_all) ++put_count; } if (bo->mem.mm_node) { - bo->mem.mm_node->private = NULL; drm_mm_put_block(bo->mem.mm_node); bo->mem.mm_node = NULL; } @@ -670,7 +669,6 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, printk(KERN_ERR TTM_PFX "Buffer eviction failed\n"); spin_lock(&glob->lru_lock); if (evict_mem.mm_node) { - evict_mem.mm_node->private = NULL; drm_mm_put_block(evict_mem.mm_node); evict_mem.mm_node = NULL; } @@ -929,8 +927,6 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, mem->mm_node = node; mem->mem_type = mem_type; mem->placement = cur_flags; - if (node) - node->private = bo; return 0; } @@ -973,7 +969,6 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, interruptible, no_wait_reserve, no_wait_gpu); if (ret == 0 && mem->mm_node) { mem->placement = cur_flags; - mem->mm_node->private = bo; return 0; } if (ret == -ERESTARTSYS) @@ -1029,7 +1024,6 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo, out_unlock: if (ret && mem.mm_node) { spin_lock(&glob->lru_lock); - mem.mm_node->private = NULL; drm_mm_put_block(mem.mm_node); spin_unlock(&glob->lru_lock); } diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 13012a1f1486..7cffb3e04232 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -353,8 +353,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, fbo->vm_node = NULL; fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); - if (fbo->mem.mm_node) - fbo->mem.mm_node->private = (void *)fbo; kref_init(&fbo->list_kref); kref_init(&fbo->kref); fbo->destroy = &ttm_transfered_destroy; diff --git a/include/drm/drm_mm.h b/include/drm/drm_mm.h index 4c10be39a43b..bf01531193d5 100644 --- a/include/drm/drm_mm.h +++ b/include/drm/drm_mm.h @@ -42,21 +42,31 @@ #endif struct drm_mm_node { - struct list_head fl_entry; - struct list_head ml_entry; - int free; + struct list_head free_stack; + struct list_head node_list; + unsigned free : 1; + unsigned scanned_block : 1; + unsigned scanned_prev_free : 1; + unsigned scanned_next_free : 1; unsigned long start; unsigned long size; struct drm_mm *mm; - void *private; }; struct drm_mm { - struct list_head fl_entry; - struct list_head ml_entry; + /* List of free memory blocks, most recently freed ordered. */ + struct list_head free_stack; + /* List of all memory nodes, ordered according to the (increasing) start + * address of the memory node. */ + struct list_head node_list; struct list_head unused_nodes; int num_unused; spinlock_t unused_lock; + unsigned scan_alignment; + unsigned long scan_size; + unsigned long scan_hit_start; + unsigned scan_hit_size; + unsigned scanned_blocks; }; /* @@ -133,6 +143,11 @@ static inline struct drm_mm *drm_get_mm(struct drm_mm_node *block) return block->mm; } +void drm_mm_init_scan(struct drm_mm *mm, unsigned long size, + unsigned alignment); +int drm_mm_scan_add_block(struct drm_mm_node *node); +int drm_mm_scan_remove_block(struct drm_mm_node *node); + extern void drm_mm_debug_table(struct drm_mm *mm, const char *prefix); #ifdef CONFIG_DEBUG_FS int drm_mm_dump_table(struct seq_file *m, struct drm_mm *mm);