2006-10-03 15:01:26 -06:00
|
|
|
/* include/asm-generic/tlb.h
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* Generic TLB shootdown code
|
|
|
|
*
|
|
|
|
* Copyright 2001 Red Hat, Inc.
|
|
|
|
* Based on code from mm/memory.c Copyright Linus Torvalds and others.
|
|
|
|
*
|
2011-05-24 18:11:45 -06:00
|
|
|
* Copyright 2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
|
|
|
|
*
|
2005-04-16 16:20:36 -06:00
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
* as published by the Free Software Foundation; either version
|
|
|
|
* 2 of the License, or (at your option) any later version.
|
|
|
|
*/
|
|
|
|
#ifndef _ASM_GENERIC__TLB_H
|
|
|
|
#define _ASM_GENERIC__TLB_H
|
|
|
|
|
|
|
|
#include <linux/swap.h>
|
2008-01-31 14:05:48 -07:00
|
|
|
#include <asm/pgalloc.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
|
2011-05-24 18:12:00 -06:00
|
|
|
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
|
|
|
|
/*
|
|
|
|
* Semi RCU freeing of the page directories.
|
|
|
|
*
|
|
|
|
* This is needed by some architectures to implement software pagetable walkers.
|
|
|
|
*
|
|
|
|
* gup_fast() and other software pagetable walkers do a lockless page-table
|
|
|
|
* walk and therefore needs some synchronization with the freeing of the page
|
|
|
|
* directories. The chosen means to accomplish that is by disabling IRQs over
|
|
|
|
* the walk.
|
|
|
|
*
|
|
|
|
* Architectures that use IPIs to flush TLBs will then automagically DTRT,
|
|
|
|
* since we unlink the page, flush TLBs, free the page. Since the disabling of
|
|
|
|
* IRQs delays the completion of the TLB flush we can never observe an already
|
|
|
|
* freed page.
|
|
|
|
*
|
|
|
|
* Architectures that do not have this (PPC) need to delay the freeing by some
|
|
|
|
* other means, this is that means.
|
|
|
|
*
|
|
|
|
* What we do is batch the freed directory pages (tables) and RCU free them.
|
|
|
|
* We use the sched RCU variant, as that guarantees that IRQ/preempt disabling
|
|
|
|
* holds off grace periods.
|
|
|
|
*
|
|
|
|
* However, in order to batch these pages we need to allocate storage, this
|
|
|
|
* allocation is deep inside the MM code and can thus easily fail on memory
|
|
|
|
* pressure. To guarantee progress we fall back to single table freeing, see
|
|
|
|
* the implementation of tlb_remove_table_one().
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
struct mmu_table_batch {
|
|
|
|
struct rcu_head rcu;
|
|
|
|
unsigned int nr;
|
|
|
|
void *tables[0];
|
|
|
|
};
|
|
|
|
|
|
|
|
#define MAX_TABLE_BATCH \
|
|
|
|
((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
|
|
|
|
|
|
|
|
extern void tlb_table_flush(struct mmu_gather *tlb);
|
|
|
|
extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
2011-05-24 18:11:45 -06:00
|
|
|
/*
|
|
|
|
* If we can't allocate a page to make a big batch of page pointers
|
|
|
|
* to work on, then just handle a few from the on-stack structure.
|
|
|
|
*/
|
|
|
|
#define MMU_GATHER_BUNDLE 8
|
|
|
|
|
2011-05-24 18:12:01 -06:00
|
|
|
struct mmu_gather_batch {
|
|
|
|
struct mmu_gather_batch *next;
|
|
|
|
unsigned int nr;
|
|
|
|
unsigned int max;
|
|
|
|
struct page *pages[0];
|
|
|
|
};
|
|
|
|
|
|
|
|
#define MAX_GATHER_BATCH \
|
|
|
|
((PAGE_SIZE - sizeof(struct mmu_gather_batch)) / sizeof(void *))
|
|
|
|
|
2013-01-04 16:35:12 -07:00
|
|
|
/*
|
|
|
|
* Limit the maximum number of mmu_gather batches to reduce a risk of soft
|
|
|
|
* lockups for non-preemptible kernels on huge machines when a lot of memory
|
|
|
|
* is zapped during unmapping.
|
|
|
|
* 10K pages freed at once should be safe even without a preemption point.
|
|
|
|
*/
|
|
|
|
#define MAX_GATHER_BATCH_COUNT (10000UL/MAX_GATHER_BATCH)
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/* struct mmu_gather is an opaque type used by the mm code for passing around
|
2005-10-29 19:16:01 -06:00
|
|
|
* any data needed by arch specific code for tlb_remove_page.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
struct mmu_gather {
|
|
|
|
struct mm_struct *mm;
|
2011-05-24 18:12:00 -06:00
|
|
|
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
|
|
|
|
struct mmu_table_batch *batch;
|
|
|
|
#endif
|
2012-06-27 19:02:21 -06:00
|
|
|
unsigned long start;
|
|
|
|
unsigned long end;
|
2011-05-24 18:12:01 -06:00
|
|
|
unsigned int need_flush : 1, /* Did free PTEs */
|
2013-04-12 17:23:54 -06:00
|
|
|
/* we are in the middle of an operation to clear
|
|
|
|
* a full mm and can make some optimizations */
|
2013-06-05 04:26:50 -06:00
|
|
|
fullmm : 1,
|
2013-04-12 17:23:54 -06:00
|
|
|
/* we have performed an operation which
|
|
|
|
* requires a complete flush of the tlb */
|
|
|
|
need_flush_all : 1;
|
2011-05-24 18:12:01 -06:00
|
|
|
|
|
|
|
struct mmu_gather_batch *active;
|
|
|
|
struct mmu_gather_batch local;
|
|
|
|
struct page *__pages[MMU_GATHER_BUNDLE];
|
2013-01-04 16:35:12 -07:00
|
|
|
unsigned int batch_count;
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
|
|
|
|
2011-05-24 18:12:14 -06:00
|
|
|
#define HAVE_GENERIC_MMU_GATHER
|
2011-05-24 18:12:01 -06:00
|
|
|
|
2011-05-24 18:12:14 -06:00
|
|
|
void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm);
|
|
|
|
void tlb_flush_mmu(struct mmu_gather *tlb);
|
2012-06-27 19:02:19 -06:00
|
|
|
void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
|
|
|
|
unsigned long end);
|
2011-05-24 18:12:14 -06:00
|
|
|
int __tlb_remove_page(struct mmu_gather *tlb, struct page *page);
|
2011-05-24 18:11:45 -06:00
|
|
|
|
|
|
|
/* tlb_remove_page
|
|
|
|
* Similar to __tlb_remove_page but will call tlb_flush_mmu() itself when
|
|
|
|
* required.
|
|
|
|
*/
|
|
|
|
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
|
|
|
|
{
|
|
|
|
if (!__tlb_remove_page(tlb, page))
|
|
|
|
tlb_flush_mmu(tlb);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation.
|
|
|
|
*
|
|
|
|
* Record the fact that pte's were really umapped in ->need_flush, so we can
|
|
|
|
* later optimise away the tlb invalidate. This helps when userspace is
|
|
|
|
* unmapping already-unmapped pages, which happens quite a lot.
|
|
|
|
*/
|
|
|
|
#define tlb_remove_tlb_entry(tlb, ptep, address) \
|
|
|
|
do { \
|
|
|
|
tlb->need_flush = 1; \
|
|
|
|
__tlb_remove_tlb_entry(tlb, ptep, address); \
|
|
|
|
} while (0)
|
|
|
|
|
2012-01-12 18:19:16 -07:00
|
|
|
/**
|
|
|
|
* tlb_remove_pmd_tlb_entry - remember a pmd mapping for later tlb invalidation
|
|
|
|
* This is a nop so far, because only x86 needs it.
|
|
|
|
*/
|
|
|
|
#ifndef __tlb_remove_pmd_tlb_entry
|
|
|
|
#define __tlb_remove_pmd_tlb_entry(tlb, pmdp, address) do {} while (0)
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \
|
|
|
|
do { \
|
|
|
|
tlb->need_flush = 1; \
|
|
|
|
__tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \
|
|
|
|
} while (0)
|
|
|
|
|
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
Upcoming paches to support the new 64-bit "BookE" powerpc architecture
will need to have the virtual address corresponding to PTE page when
freeing it, due to the way the HW table walker works.
Basically, the TLB can be loaded with "large" pages that cover the whole
virtual space (well, sort-of, half of it actually) represented by a PTE
page, and which contain an "indirect" bit indicating that this TLB entry
RPN points to an array of PTEs from which the TLB can then create direct
entries. Thus, in order to invalidate those when PTE pages are deleted,
we need the virtual address to pass to tlbilx or tlbivax instructions.
The old trick of sticking it somewhere in the PTE page struct page sucks
too much, the address is almost readily available in all call sites and
almost everybody implemets these as macros, so we may as well add the
argument everywhere. I added it to the pmd and pud variants for consistency.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: David Howells <dhowells@redhat.com> [MN10300 & FRV]
Acked-by: Nick Piggin <npiggin@suse.de>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [s390]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-07-21 23:44:28 -06:00
|
|
|
#define pte_free_tlb(tlb, ptep, address) \
|
2005-04-16 16:20:36 -06:00
|
|
|
do { \
|
|
|
|
tlb->need_flush = 1; \
|
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
Upcoming paches to support the new 64-bit "BookE" powerpc architecture
will need to have the virtual address corresponding to PTE page when
freeing it, due to the way the HW table walker works.
Basically, the TLB can be loaded with "large" pages that cover the whole
virtual space (well, sort-of, half of it actually) represented by a PTE
page, and which contain an "indirect" bit indicating that this TLB entry
RPN points to an array of PTEs from which the TLB can then create direct
entries. Thus, in order to invalidate those when PTE pages are deleted,
we need the virtual address to pass to tlbilx or tlbivax instructions.
The old trick of sticking it somewhere in the PTE page struct page sucks
too much, the address is almost readily available in all call sites and
almost everybody implemets these as macros, so we may as well add the
argument everywhere. I added it to the pmd and pud variants for consistency.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: David Howells <dhowells@redhat.com> [MN10300 & FRV]
Acked-by: Nick Piggin <npiggin@suse.de>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [s390]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-07-21 23:44:28 -06:00
|
|
|
__pte_free_tlb(tlb, ptep, address); \
|
2005-04-16 16:20:36 -06:00
|
|
|
} while (0)
|
|
|
|
|
|
|
|
#ifndef __ARCH_HAS_4LEVEL_HACK
|
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
Upcoming paches to support the new 64-bit "BookE" powerpc architecture
will need to have the virtual address corresponding to PTE page when
freeing it, due to the way the HW table walker works.
Basically, the TLB can be loaded with "large" pages that cover the whole
virtual space (well, sort-of, half of it actually) represented by a PTE
page, and which contain an "indirect" bit indicating that this TLB entry
RPN points to an array of PTEs from which the TLB can then create direct
entries. Thus, in order to invalidate those when PTE pages are deleted,
we need the virtual address to pass to tlbilx or tlbivax instructions.
The old trick of sticking it somewhere in the PTE page struct page sucks
too much, the address is almost readily available in all call sites and
almost everybody implemets these as macros, so we may as well add the
argument everywhere. I added it to the pmd and pud variants for consistency.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: David Howells <dhowells@redhat.com> [MN10300 & FRV]
Acked-by: Nick Piggin <npiggin@suse.de>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [s390]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-07-21 23:44:28 -06:00
|
|
|
#define pud_free_tlb(tlb, pudp, address) \
|
2005-04-16 16:20:36 -06:00
|
|
|
do { \
|
|
|
|
tlb->need_flush = 1; \
|
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
Upcoming paches to support the new 64-bit "BookE" powerpc architecture
will need to have the virtual address corresponding to PTE page when
freeing it, due to the way the HW table walker works.
Basically, the TLB can be loaded with "large" pages that cover the whole
virtual space (well, sort-of, half of it actually) represented by a PTE
page, and which contain an "indirect" bit indicating that this TLB entry
RPN points to an array of PTEs from which the TLB can then create direct
entries. Thus, in order to invalidate those when PTE pages are deleted,
we need the virtual address to pass to tlbilx or tlbivax instructions.
The old trick of sticking it somewhere in the PTE page struct page sucks
too much, the address is almost readily available in all call sites and
almost everybody implemets these as macros, so we may as well add the
argument everywhere. I added it to the pmd and pud variants for consistency.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: David Howells <dhowells@redhat.com> [MN10300 & FRV]
Acked-by: Nick Piggin <npiggin@suse.de>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [s390]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-07-21 23:44:28 -06:00
|
|
|
__pud_free_tlb(tlb, pudp, address); \
|
2005-04-16 16:20:36 -06:00
|
|
|
} while (0)
|
|
|
|
#endif
|
|
|
|
|
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
Upcoming paches to support the new 64-bit "BookE" powerpc architecture
will need to have the virtual address corresponding to PTE page when
freeing it, due to the way the HW table walker works.
Basically, the TLB can be loaded with "large" pages that cover the whole
virtual space (well, sort-of, half of it actually) represented by a PTE
page, and which contain an "indirect" bit indicating that this TLB entry
RPN points to an array of PTEs from which the TLB can then create direct
entries. Thus, in order to invalidate those when PTE pages are deleted,
we need the virtual address to pass to tlbilx or tlbivax instructions.
The old trick of sticking it somewhere in the PTE page struct page sucks
too much, the address is almost readily available in all call sites and
almost everybody implemets these as macros, so we may as well add the
argument everywhere. I added it to the pmd and pud variants for consistency.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: David Howells <dhowells@redhat.com> [MN10300 & FRV]
Acked-by: Nick Piggin <npiggin@suse.de>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [s390]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-07-21 23:44:28 -06:00
|
|
|
#define pmd_free_tlb(tlb, pmdp, address) \
|
2005-04-16 16:20:36 -06:00
|
|
|
do { \
|
|
|
|
tlb->need_flush = 1; \
|
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
Upcoming paches to support the new 64-bit "BookE" powerpc architecture
will need to have the virtual address corresponding to PTE page when
freeing it, due to the way the HW table walker works.
Basically, the TLB can be loaded with "large" pages that cover the whole
virtual space (well, sort-of, half of it actually) represented by a PTE
page, and which contain an "indirect" bit indicating that this TLB entry
RPN points to an array of PTEs from which the TLB can then create direct
entries. Thus, in order to invalidate those when PTE pages are deleted,
we need the virtual address to pass to tlbilx or tlbivax instructions.
The old trick of sticking it somewhere in the PTE page struct page sucks
too much, the address is almost readily available in all call sites and
almost everybody implemets these as macros, so we may as well add the
argument everywhere. I added it to the pmd and pud variants for consistency.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: David Howells <dhowells@redhat.com> [MN10300 & FRV]
Acked-by: Nick Piggin <npiggin@suse.de>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [s390]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-07-21 23:44:28 -06:00
|
|
|
__pmd_free_tlb(tlb, pmdp, address); \
|
2005-04-16 16:20:36 -06:00
|
|
|
} while (0)
|
|
|
|
|
|
|
|
#define tlb_migrate_finish(mm) do {} while (0)
|
|
|
|
|
|
|
|
#endif /* _ASM_GENERIC__TLB_H */
|