MIPS: Put PGD in C0_CONTEXT for 64-bit R2 processors.
Processors that support the mips64r2 ISA can in four instructions convert a shifted PGD pointer stored in the upper bits of c0_context into a usable pointer. By doing this we save a memory load and associated potential cache miss in the TLB exception handlers. Since the upper bits of c0_context were holding the CPU number, we move this to the upper bits of c0_xcontext which doesn't have enough bits to hold the PGD pointer, but has plenty for the CPU number. Signed-off-by: David Daney <ddaney@caviumnetworks.com> Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
This commit is contained in:
parent
92078e0618
commit
82622284dd
|
@ -1427,6 +1427,9 @@ config CPU_SUPPORTS_64BIT_KERNEL
|
||||||
bool
|
bool
|
||||||
config CPU_SUPPORTS_HUGEPAGES
|
config CPU_SUPPORTS_HUGEPAGES
|
||||||
bool
|
bool
|
||||||
|
config MIPS_PGD_C0_CONTEXT
|
||||||
|
bool
|
||||||
|
default y if 64BIT && CPU_MIPSR2
|
||||||
|
|
||||||
#
|
#
|
||||||
# Set to y for ptrace access to watch registers.
|
# Set to y for ptrace access to watch registers.
|
||||||
|
|
|
@ -24,6 +24,33 @@
|
||||||
#endif /* SMTC */
|
#endif /* SMTC */
|
||||||
#include <asm-generic/mm_hooks.h>
|
#include <asm-generic/mm_hooks.h>
|
||||||
|
|
||||||
|
#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
|
||||||
|
|
||||||
|
#define TLBMISS_HANDLER_SETUP_PGD(pgd) \
|
||||||
|
tlbmiss_handler_setup_pgd((unsigned long)(pgd))
|
||||||
|
|
||||||
|
static inline void tlbmiss_handler_setup_pgd(unsigned long pgd)
|
||||||
|
{
|
||||||
|
/* Check for swapper_pg_dir and convert to physical address. */
|
||||||
|
if ((pgd & CKSEG3) == CKSEG0)
|
||||||
|
pgd = CPHYSADDR(pgd);
|
||||||
|
write_c0_context(pgd << 11);
|
||||||
|
}
|
||||||
|
|
||||||
|
#define TLBMISS_HANDLER_SETUP() \
|
||||||
|
do { \
|
||||||
|
TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir); \
|
||||||
|
write_c0_xcontext((unsigned long) smp_processor_id() << 51); \
|
||||||
|
} while (0)
|
||||||
|
|
||||||
|
|
||||||
|
static inline unsigned long get_current_pgd(void)
|
||||||
|
{
|
||||||
|
return PHYS_TO_XKSEG_CACHED((read_c0_context() >> 11) & ~0xfffUL);
|
||||||
|
}
|
||||||
|
|
||||||
|
#else /* CONFIG_MIPS_PGD_C0_CONTEXT: using pgd_current*/
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* For the fast tlb miss handlers, we keep a per cpu array of pointers
|
* For the fast tlb miss handlers, we keep a per cpu array of pointers
|
||||||
* to the current pgd for each processor. Also, the proc. id is stuffed
|
* to the current pgd for each processor. Also, the proc. id is stuffed
|
||||||
|
@ -46,7 +73,7 @@ extern unsigned long pgd_current[];
|
||||||
back_to_back_c0_hazard(); \
|
back_to_back_c0_hazard(); \
|
||||||
TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir)
|
TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir)
|
||||||
#endif
|
#endif
|
||||||
|
#endif /* CONFIG_MIPS_PGD_C0_CONTEXT*/
|
||||||
#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX)
|
#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX)
|
||||||
|
|
||||||
#define ASID_INC 0x40
|
#define ASID_INC 0x40
|
||||||
|
|
|
@ -87,15 +87,19 @@
|
||||||
#ifdef CONFIG_SMP
|
#ifdef CONFIG_SMP
|
||||||
#ifdef CONFIG_MIPS_MT_SMTC
|
#ifdef CONFIG_MIPS_MT_SMTC
|
||||||
#define PTEBASE_SHIFT 19 /* TCBIND */
|
#define PTEBASE_SHIFT 19 /* TCBIND */
|
||||||
|
#define CPU_ID_REG CP0_TCBIND
|
||||||
|
#define CPU_ID_MFC0 mfc0
|
||||||
|
#elif defined(CONFIG_MIPS_PGD_C0_CONTEXT)
|
||||||
|
#define PTEBASE_SHIFT 48 /* XCONTEXT */
|
||||||
|
#define CPU_ID_REG CP0_XCONTEXT
|
||||||
|
#define CPU_ID_MFC0 MFC0
|
||||||
#else
|
#else
|
||||||
#define PTEBASE_SHIFT 23 /* CONTEXT */
|
#define PTEBASE_SHIFT 23 /* CONTEXT */
|
||||||
|
#define CPU_ID_REG CP0_CONTEXT
|
||||||
|
#define CPU_ID_MFC0 MFC0
|
||||||
#endif
|
#endif
|
||||||
.macro get_saved_sp /* SMP variation */
|
.macro get_saved_sp /* SMP variation */
|
||||||
#ifdef CONFIG_MIPS_MT_SMTC
|
CPU_ID_MFC0 k0, CPU_ID_REG
|
||||||
mfc0 k0, CP0_TCBIND
|
|
||||||
#else
|
|
||||||
MFC0 k0, CP0_CONTEXT
|
|
||||||
#endif
|
|
||||||
#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
|
#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
|
||||||
lui k1, %hi(kernelsp)
|
lui k1, %hi(kernelsp)
|
||||||
#else
|
#else
|
||||||
|
@ -111,11 +115,7 @@
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
.macro set_saved_sp stackp temp temp2
|
.macro set_saved_sp stackp temp temp2
|
||||||
#ifdef CONFIG_MIPS_MT_SMTC
|
CPU_ID_MFC0 \temp, CPU_ID_REG
|
||||||
mfc0 \temp, CP0_TCBIND
|
|
||||||
#else
|
|
||||||
MFC0 \temp, CP0_CONTEXT
|
|
||||||
#endif
|
|
||||||
LONG_SRL \temp, PTEBASE_SHIFT
|
LONG_SRL \temp, PTEBASE_SHIFT
|
||||||
LONG_S \stackp, kernelsp(\temp)
|
LONG_S \stackp, kernelsp(\temp)
|
||||||
.endm
|
.endm
|
||||||
|
|
|
@ -462,7 +462,9 @@ void __init_refok free_initmem(void)
|
||||||
__pa_symbol(&__init_end));
|
__pa_symbol(&__init_end));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
|
||||||
unsigned long pgd_current[NR_CPUS];
|
unsigned long pgd_current[NR_CPUS];
|
||||||
|
#endif
|
||||||
/*
|
/*
|
||||||
* On 64-bit we've got three-level pagetables with a slightly
|
* On 64-bit we've got three-level pagetables with a slightly
|
||||||
* different layout ...
|
* different layout ...
|
||||||
|
|
|
@ -160,6 +160,12 @@ static u32 tlb_handler[128] __cpuinitdata;
|
||||||
static struct uasm_label labels[128] __cpuinitdata;
|
static struct uasm_label labels[128] __cpuinitdata;
|
||||||
static struct uasm_reloc relocs[128] __cpuinitdata;
|
static struct uasm_reloc relocs[128] __cpuinitdata;
|
||||||
|
|
||||||
|
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
|
||||||
|
/*
|
||||||
|
* CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
|
||||||
|
* we cannot do r3000 under these circumstances.
|
||||||
|
*/
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* The R3000 TLB handler is simple.
|
* The R3000 TLB handler is simple.
|
||||||
*/
|
*/
|
||||||
|
@ -199,6 +205,7 @@ static void __cpuinit build_r3000_tlb_refill_handler(void)
|
||||||
|
|
||||||
dump_handler((u32 *)ebase, 32);
|
dump_handler((u32 *)ebase, 32);
|
||||||
}
|
}
|
||||||
|
#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* The R4000 TLB handler is much more complicated. We have two
|
* The R4000 TLB handler is much more complicated. We have two
|
||||||
|
@ -497,8 +504,9 @@ static void __cpuinit
|
||||||
build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
|
build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
|
||||||
unsigned int tmp, unsigned int ptr)
|
unsigned int tmp, unsigned int ptr)
|
||||||
{
|
{
|
||||||
|
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
|
||||||
long pgdc = (long)pgd_current;
|
long pgdc = (long)pgd_current;
|
||||||
|
#endif
|
||||||
/*
|
/*
|
||||||
* The vmalloc handling is not in the hotpath.
|
* The vmalloc handling is not in the hotpath.
|
||||||
*/
|
*/
|
||||||
|
@ -506,7 +514,15 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
|
||||||
uasm_il_bltz(p, r, tmp, label_vmalloc);
|
uasm_il_bltz(p, r, tmp, label_vmalloc);
|
||||||
/* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
|
/* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
|
||||||
|
|
||||||
#ifdef CONFIG_SMP
|
#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
|
||||||
|
/*
|
||||||
|
* &pgd << 11 stored in CONTEXT [23..63].
|
||||||
|
*/
|
||||||
|
UASM_i_MFC0(p, ptr, C0_CONTEXT);
|
||||||
|
uasm_i_dins(p, ptr, 0, 0, 23); /* Clear lower 23 bits of context. */
|
||||||
|
uasm_i_ori(p, ptr, ptr, 0x540); /* 1 0 1 0 1 << 6 xkphys cached */
|
||||||
|
uasm_i_drotr(p, ptr, ptr, 11);
|
||||||
|
#elif defined(CONFIG_SMP)
|
||||||
# ifdef CONFIG_MIPS_MT_SMTC
|
# ifdef CONFIG_MIPS_MT_SMTC
|
||||||
/*
|
/*
|
||||||
* SMTC uses TCBind value as "CPU" index
|
* SMTC uses TCBind value as "CPU" index
|
||||||
|
@ -520,7 +536,7 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
|
||||||
*/
|
*/
|
||||||
uasm_i_dmfc0(p, ptr, C0_CONTEXT);
|
uasm_i_dmfc0(p, ptr, C0_CONTEXT);
|
||||||
uasm_i_dsrl(p, ptr, ptr, 23);
|
uasm_i_dsrl(p, ptr, ptr, 23);
|
||||||
#endif
|
# endif
|
||||||
UASM_i_LA_mostly(p, tmp, pgdc);
|
UASM_i_LA_mostly(p, tmp, pgdc);
|
||||||
uasm_i_daddu(p, ptr, ptr, tmp);
|
uasm_i_daddu(p, ptr, ptr, tmp);
|
||||||
uasm_i_dmfc0(p, tmp, C0_BADVADDR);
|
uasm_i_dmfc0(p, tmp, C0_BADVADDR);
|
||||||
|
@ -1033,6 +1049,7 @@ build_pte_modifiable(u32 **p, struct uasm_reloc **r,
|
||||||
iPTE_LW(p, pte, ptr);
|
iPTE_LW(p, pte, ptr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
|
||||||
/*
|
/*
|
||||||
* R3000 style TLB load/store/modify handlers.
|
* R3000 style TLB load/store/modify handlers.
|
||||||
*/
|
*/
|
||||||
|
@ -1184,6 +1201,7 @@ static void __cpuinit build_r3000_tlb_modify_handler(void)
|
||||||
|
|
||||||
dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
|
dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
|
||||||
}
|
}
|
||||||
|
#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* R4000 style TLB load/store/modify handlers.
|
* R4000 style TLB load/store/modify handlers.
|
||||||
|
@ -1400,6 +1418,7 @@ void __cpuinit build_tlb_refill_handler(void)
|
||||||
case CPU_TX3912:
|
case CPU_TX3912:
|
||||||
case CPU_TX3922:
|
case CPU_TX3922:
|
||||||
case CPU_TX3927:
|
case CPU_TX3927:
|
||||||
|
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
|
||||||
build_r3000_tlb_refill_handler();
|
build_r3000_tlb_refill_handler();
|
||||||
if (!run_once) {
|
if (!run_once) {
|
||||||
build_r3000_tlb_load_handler();
|
build_r3000_tlb_load_handler();
|
||||||
|
@ -1407,6 +1426,9 @@ void __cpuinit build_tlb_refill_handler(void)
|
||||||
build_r3000_tlb_modify_handler();
|
build_r3000_tlb_modify_handler();
|
||||||
run_once++;
|
run_once++;
|
||||||
}
|
}
|
||||||
|
#else
|
||||||
|
panic("No R3000 TLB refill handler");
|
||||||
|
#endif
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case CPU_R6000:
|
case CPU_R6000:
|
||||||
|
|
Loading…
Reference in a new issue