2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* This file is subject to the terms and conditions of the GNU General Public
|
|
|
|
* License. See the file "COPYING" in the main directory of this archive
|
|
|
|
* for more details.
|
|
|
|
*
|
|
|
|
* Synthesize TLB refill handlers at runtime.
|
|
|
|
*
|
2013-01-22 04:59:30 -07:00
|
|
|
* Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
|
|
|
|
* Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki
|
2006-04-05 02:45:45 -06:00
|
|
|
* Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
|
2009-05-27 18:47:44 -06:00
|
|
|
* Copyright (C) 2008, 2009 Cavium Networks, Inc.
|
2012-07-06 15:56:00 -06:00
|
|
|
* Copyright (C) 2011 MIPS Technologies, Inc.
|
2006-04-05 02:45:45 -06:00
|
|
|
*
|
|
|
|
* ... and the days got worse and worse and now you see
|
2016-02-25 01:44:58 -07:00
|
|
|
* I've gone completely out of my mind.
|
2006-04-05 02:45:45 -06:00
|
|
|
*
|
|
|
|
* They're coming to take me a away haha
|
|
|
|
* they're coming to take me a away hoho hihi haha
|
|
|
|
* to the funny farm where code is beautiful all the time ...
|
|
|
|
*
|
|
|
|
* (Condolences to Napoleon XIV)
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
|
2009-05-20 12:40:59 -06:00
|
|
|
#include <linux/bug.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/types.h>
|
2009-06-19 07:05:26 -06:00
|
|
|
#include <linux/smp.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/string.h>
|
2010-12-21 15:19:11 -07:00
|
|
|
#include <linux/cache.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2010-12-21 15:19:11 -07:00
|
|
|
#include <asm/cacheflush.h>
|
2013-09-17 02:25:47 -06:00
|
|
|
#include <asm/cpu-type.h>
|
2010-12-21 15:19:11 -07:00
|
|
|
#include <asm/pgtable.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <asm/war.h>
|
2010-01-28 07:21:24 -07:00
|
|
|
#include <asm/uasm.h>
|
2012-03-28 11:30:02 -06:00
|
|
|
#include <asm/setup.h>
|
2008-01-28 13:05:38 -07:00
|
|
|
|
2015-04-27 16:47:59 -06:00
|
|
|
static int mips_xpa_disabled;
|
2015-02-26 17:16:38 -07:00
|
|
|
|
|
|
|
static int __init xpa_disable(char *s)
|
|
|
|
{
|
|
|
|
mips_xpa_disabled = 1;
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
__setup("noxpa", xpa_disable);
|
|
|
|
|
2010-04-28 13:16:18 -06:00
|
|
|
/*
|
|
|
|
* TLB load/store/modify handlers.
|
|
|
|
*
|
|
|
|
* Only the fastpath gets synthesized at runtime, the slowpath for
|
|
|
|
* do_page_fault remains normal asm.
|
|
|
|
*/
|
|
|
|
extern void tlb_do_page_fault_0(void);
|
|
|
|
extern void tlb_do_page_fault_1(void);
|
|
|
|
|
2011-07-05 17:34:46 -06:00
|
|
|
struct work_registers {
|
|
|
|
int r1;
|
|
|
|
int r2;
|
|
|
|
int r3;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct tlb_reg_save {
|
|
|
|
unsigned long a;
|
|
|
|
unsigned long b;
|
|
|
|
} ____cacheline_aligned_in_smp;
|
|
|
|
|
|
|
|
static struct tlb_reg_save handler_reg_save[NR_CPUS];
|
2010-04-28 13:16:18 -06:00
|
|
|
|
2007-10-11 16:46:14 -06:00
|
|
|
static inline int r45k_bvahwbug(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
/* XXX: We should probe for the presence of this bug, but we don't. */
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-10-11 16:46:14 -06:00
|
|
|
static inline int r4k_250MHZhwbug(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
/* XXX: We should probe for the presence of this bug, but we don't. */
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-10-11 16:46:14 -06:00
|
|
|
static inline int __maybe_unused bcm1250_m3_war(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
return BCM1250_M3_WAR;
|
|
|
|
}
|
|
|
|
|
2007-10-11 16:46:14 -06:00
|
|
|
static inline int __maybe_unused r10000_llsc_war(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
return R10000_LLSC_WAR;
|
|
|
|
}
|
|
|
|
|
2010-12-20 16:54:50 -07:00
|
|
|
static int use_bbit_insns(void)
|
|
|
|
{
|
|
|
|
switch (current_cpu_type()) {
|
|
|
|
case CPU_CAVIUM_OCTEON:
|
|
|
|
case CPU_CAVIUM_OCTEON_PLUS:
|
|
|
|
case CPU_CAVIUM_OCTEON2:
|
2013-07-29 16:07:03 -06:00
|
|
|
case CPU_CAVIUM_OCTEON3:
|
2010-12-20 16:54:50 -07:00
|
|
|
return 1;
|
|
|
|
default:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-12-27 19:07:57 -07:00
|
|
|
static int use_lwx_insns(void)
|
|
|
|
{
|
|
|
|
switch (current_cpu_type()) {
|
|
|
|
case CPU_CAVIUM_OCTEON2:
|
2013-07-29 16:07:03 -06:00
|
|
|
case CPU_CAVIUM_OCTEON3:
|
2010-12-27 19:07:57 -07:00
|
|
|
return 1;
|
|
|
|
default:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \
|
|
|
|
CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0
|
|
|
|
static bool scratchpad_available(void)
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
static int scratchpad_offset(int i)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* CVMSEG starts at address -32768 and extends for
|
|
|
|
* CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines.
|
|
|
|
*/
|
|
|
|
i += 1; /* Kernel use starts at the top and works down. */
|
|
|
|
return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static bool scratchpad_available(void)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
static int scratchpad_offset(int i)
|
|
|
|
{
|
|
|
|
BUG();
|
2011-01-19 16:24:42 -07:00
|
|
|
/* Really unreachable, but evidently some GCC want this. */
|
|
|
|
return 0;
|
2010-12-27 19:07:57 -07:00
|
|
|
}
|
|
|
|
#endif
|
2006-08-23 07:26:50 -06:00
|
|
|
/*
|
|
|
|
* Found by experiment: At least some revisions of the 4kc throw under
|
|
|
|
* some circumstances a machine check exception, triggered by invalid
|
|
|
|
* values in the index register. Delaying the tlbp instruction until
|
|
|
|
* after the next branch, plus adding an additional nop in front of
|
|
|
|
* tlbwi/tlbwr avoids the invalid index register values. Nobody knows
|
|
|
|
* why; it's not an issue caused by the core RTL.
|
|
|
|
*
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static int m4kc_tlbp_war(void)
|
2006-08-23 07:26:50 -06:00
|
|
|
{
|
|
|
|
return (current_cpu_data.processor_id & 0xffff00) ==
|
|
|
|
(PRID_COMP_MIPS | PRID_IMP_4KC);
|
|
|
|
}
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
/* Handle labels (which must be positive integers). */
|
2005-04-16 16:20:36 -06:00
|
|
|
enum label_id {
|
2008-01-28 13:05:38 -07:00
|
|
|
label_second_part = 1,
|
2005-04-16 16:20:36 -06:00
|
|
|
label_leave,
|
|
|
|
label_vmalloc,
|
|
|
|
label_vmalloc_done,
|
2012-10-13 14:46:26 -06:00
|
|
|
label_tlbw_hazard_0,
|
|
|
|
label_split = label_tlbw_hazard_0 + 8,
|
2010-02-10 16:12:47 -07:00
|
|
|
label_tlbl_goaround1,
|
|
|
|
label_tlbl_goaround2,
|
2005-04-16 16:20:36 -06:00
|
|
|
label_nopage_tlbl,
|
|
|
|
label_nopage_tlbs,
|
|
|
|
label_nopage_tlbm,
|
|
|
|
label_smp_pgtable_change,
|
|
|
|
label_r3000_write_probe_fail,
|
2010-04-28 13:16:18 -06:00
|
|
|
label_large_segbits_fault,
|
2012-10-16 16:48:10 -06:00
|
|
|
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
|
2009-05-27 18:47:44 -06:00
|
|
|
label_tlb_huge_update,
|
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
UASM_L_LA(_second_part)
|
|
|
|
UASM_L_LA(_leave)
|
|
|
|
UASM_L_LA(_vmalloc)
|
|
|
|
UASM_L_LA(_vmalloc_done)
|
2012-10-13 14:46:26 -06:00
|
|
|
/* _tlbw_hazard_x is handled differently. */
|
2008-01-28 13:05:38 -07:00
|
|
|
UASM_L_LA(_split)
|
2010-02-10 16:12:47 -07:00
|
|
|
UASM_L_LA(_tlbl_goaround1)
|
|
|
|
UASM_L_LA(_tlbl_goaround2)
|
2008-01-28 13:05:38 -07:00
|
|
|
UASM_L_LA(_nopage_tlbl)
|
|
|
|
UASM_L_LA(_nopage_tlbs)
|
|
|
|
UASM_L_LA(_nopage_tlbm)
|
|
|
|
UASM_L_LA(_smp_pgtable_change)
|
|
|
|
UASM_L_LA(_r3000_write_probe_fail)
|
2010-04-28 13:16:18 -06:00
|
|
|
UASM_L_LA(_large_segbits_fault)
|
2012-10-16 16:48:10 -06:00
|
|
|
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
|
2009-05-27 18:47:44 -06:00
|
|
|
UASM_L_LA(_tlb_huge_update)
|
|
|
|
#endif
|
[MIPS] Load modules to CKSEG0 if CONFIG_BUILD_ELF64=n
This is a patch to load 64-bit modules to CKSEG0 so that can be
compiled with -msym32 option. This makes each module ~10% smaller.
* introduce MODULE_START and MODULE_END
* custom module_alloc()
* PGD for modules
* change XTLB refill handler synthesizer
* enable -msym32 for modules again
(revert ca78b1a5c6a6e70e052d3ea253828e49b5d07c8a)
New XTLB refill handler looks like this:
80000080 dmfc0 k0,C0_BADVADDR
80000084 bltz k0,800000e4 # goto l_module_alloc
80000088 lui k1,0x8046 # %high(pgd_current)
8000008c ld k1,24600(k1) # %low(pgd_current)
80000090 dsrl k0,k0,0x1b # l_vmalloc_done:
80000094 andi k0,k0,0x1ff8
80000098 daddu k1,k1,k0
8000009c dmfc0 k0,C0_BADVADDR
800000a0 ld k1,0(k1)
800000a4 dsrl k0,k0,0x12
800000a8 andi k0,k0,0xff8
800000ac daddu k1,k1,k0
800000b0 dmfc0 k0,C0_XCONTEXT
800000b4 ld k1,0(k1)
800000b8 andi k0,k0,0xff0
800000bc daddu k1,k1,k0
800000c0 ld k0,0(k1)
800000c4 ld k1,8(k1)
800000c8 dsrl k0,k0,0x6
800000cc mtc0 k0,C0_ENTRYLO0
800000d0 dsrl k1,k1,0x6
800000d4 mtc0 k1,C0_ENTRYL01
800000d8 nop
800000dc tlbwr
800000e0 eret
800000e4 dsll k1,k0,0x2 # l_module_alloc:
800000e8 bgez k1,80000008 # goto l_vmalloc
800000ec lui k1,0xc000
800000f0 dsubu k0,k0,k1
800000f4 lui k1,0x8046 # %high(module_pg_dir)
800000f8 beq zero,zero,80000000
800000fc nop
80000000 beq zero,zero,80000090 # goto l_vmalloc_done
80000004 daddiu k1,k1,0x4000
80000008 dsll32 k1,k1,0x0 # l_vmalloc:
8000000c dsubu k0,k0,k1
80000010 beq zero,zero,80000090 # goto l_vmalloc_done
80000014 lui k1,0x8046 # %high(swapper_pg_dir)
Signed-off-by: Atsushi Nemoto <anemo@mba.ocn.ne.jp>
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2006-10-25 09:08:31 -06:00
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static int hazard_instance;
|
2012-10-13 14:46:26 -06:00
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance)
|
2012-10-13 14:46:26 -06:00
|
|
|
{
|
|
|
|
switch (instance) {
|
|
|
|
case 0 ... 7:
|
|
|
|
uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance);
|
|
|
|
return;
|
|
|
|
default:
|
|
|
|
BUG();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance)
|
2012-10-13 14:46:26 -06:00
|
|
|
{
|
|
|
|
switch (instance) {
|
|
|
|
case 0 ... 7:
|
|
|
|
uasm_build_label(l, *p, label_tlbw_hazard_0 + instance);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
BUG();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-10-18 01:11:17 -06:00
|
|
|
/*
|
2012-10-16 14:20:26 -06:00
|
|
|
* pgtable bits are assigned dynamically depending on processor feature
|
|
|
|
* and statically based on kernel configuration. This spits out the actual
|
2013-01-22 04:59:30 -07:00
|
|
|
* values the kernel is using. Required to make sense from disassembled
|
2012-10-16 14:20:26 -06:00
|
|
|
* TLB exception handlers.
|
2007-10-18 01:11:17 -06:00
|
|
|
*/
|
2012-10-16 14:20:26 -06:00
|
|
|
static void output_pgtable_bits_defines(void)
|
|
|
|
{
|
|
|
|
#define pr_define(fmt, ...) \
|
|
|
|
pr_debug("#define " fmt, ##__VA_ARGS__)
|
|
|
|
|
|
|
|
pr_debug("#include <asm/asm.h>\n");
|
|
|
|
pr_debug("#include <asm/regdef.h>\n");
|
|
|
|
pr_debug("\n");
|
|
|
|
|
|
|
|
pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT);
|
2016-04-19 02:25:03 -06:00
|
|
|
pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT);
|
2012-10-16 14:20:26 -06:00
|
|
|
pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT);
|
|
|
|
pr_define("_PAGE_ACCESSED_SHIFT %d\n", _PAGE_ACCESSED_SHIFT);
|
|
|
|
pr_define("_PAGE_MODIFIED_SHIFT %d\n", _PAGE_MODIFIED_SHIFT);
|
2012-10-18 05:54:15 -06:00
|
|
|
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
|
2012-10-16 14:20:26 -06:00
|
|
|
pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT);
|
|
|
|
#endif
|
|
|
|
#ifdef _PAGE_NO_EXEC_SHIFT
|
2016-04-19 02:25:03 -06:00
|
|
|
if (cpu_has_rixi)
|
2012-10-16 14:20:26 -06:00
|
|
|
pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT);
|
2015-02-26 17:16:37 -07:00
|
|
|
#endif
|
2012-10-16 14:20:26 -06:00
|
|
|
pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT);
|
|
|
|
pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT);
|
|
|
|
pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT);
|
|
|
|
pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT);
|
|
|
|
pr_debug("\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void dump_handler(const char *symbol, const u32 *handler, int count)
|
2007-10-18 01:11:17 -06:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2012-10-16 14:20:26 -06:00
|
|
|
pr_debug("LEAF(%s)\n", symbol);
|
|
|
|
|
2007-10-18 01:11:17 -06:00
|
|
|
pr_debug("\t.set push\n");
|
|
|
|
pr_debug("\t.set noreorder\n");
|
|
|
|
|
|
|
|
for (i = 0; i < count; i++)
|
2012-10-16 14:20:26 -06:00
|
|
|
pr_debug("\t.word\t0x%08x\t\t# %p\n", handler[i], &handler[i]);
|
2007-10-18 01:11:17 -06:00
|
|
|
|
2012-10-16 14:20:26 -06:00
|
|
|
pr_debug("\t.set\tpop\n");
|
|
|
|
|
|
|
|
pr_debug("\tEND(%s)\n", symbol);
|
2007-10-18 01:11:17 -06:00
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/* The only general purpose registers allowed in TLB handlers. */
|
|
|
|
#define K0 26
|
|
|
|
#define K1 27
|
|
|
|
|
|
|
|
/* Some CP0 registers */
|
2006-04-05 02:45:45 -06:00
|
|
|
#define C0_INDEX 0, 0
|
|
|
|
#define C0_ENTRYLO0 2, 0
|
|
|
|
#define C0_TCBIND 2, 2
|
|
|
|
#define C0_ENTRYLO1 3, 0
|
|
|
|
#define C0_CONTEXT 4, 0
|
2009-05-27 18:47:44 -06:00
|
|
|
#define C0_PAGEMASK 5, 0
|
2016-03-02 18:45:12 -07:00
|
|
|
#define C0_PWBASE 5, 5
|
|
|
|
#define C0_PWFIELD 5, 6
|
|
|
|
#define C0_PWSIZE 5, 7
|
|
|
|
#define C0_PWCTL 6, 6
|
2006-04-05 02:45:45 -06:00
|
|
|
#define C0_BADVADDR 8, 0
|
2016-03-02 18:45:12 -07:00
|
|
|
#define C0_PGD 9, 7
|
2006-04-05 02:45:45 -06:00
|
|
|
#define C0_ENTRYHI 10, 0
|
|
|
|
#define C0_EPC 14, 0
|
|
|
|
#define C0_XCONTEXT 20, 0
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2005-09-03 16:56:16 -06:00
|
|
|
#ifdef CONFIG_64BIT
|
2008-01-28 13:05:38 -07:00
|
|
|
# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT)
|
2005-04-16 16:20:36 -06:00
|
|
|
#else
|
2008-01-28 13:05:38 -07:00
|
|
|
# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT)
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
|
|
|
|
/* The worst case length of the handler is around 18 instructions for
|
|
|
|
* R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
|
|
|
|
* Maximum space available is 32 instructions for R3000 and 64
|
|
|
|
* instructions for R4000.
|
|
|
|
*
|
|
|
|
* We deliberately chose a buffer size of 128, so we won't scribble
|
|
|
|
* over anything important on overflow before we panic.
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static u32 tlb_handler[128];
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* simply assume worst case size for labels and relocs */
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static struct uasm_label labels[128];
|
|
|
|
static struct uasm_reloc relocs[128];
|
2005-04-16 16:20:36 -06:00
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static int check_for_high_segbits;
|
2015-09-22 12:42:52 -06:00
|
|
|
static bool fill_includes_sw_bits;
|
2010-12-21 15:19:11 -07:00
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static unsigned int kscratch_used_mask;
|
2010-12-21 15:19:11 -07:00
|
|
|
|
2013-06-11 08:41:35 -06:00
|
|
|
static inline int __maybe_unused c0_kscratch(void)
|
|
|
|
{
|
|
|
|
switch (current_cpu_type()) {
|
|
|
|
case CPU_XLP:
|
|
|
|
case CPU_XLR:
|
|
|
|
return 22;
|
|
|
|
default:
|
|
|
|
return 31;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static int allocate_kscratch(void)
|
2010-12-21 15:19:11 -07:00
|
|
|
{
|
|
|
|
int r;
|
|
|
|
unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask;
|
|
|
|
|
|
|
|
r = ffs(a);
|
|
|
|
|
|
|
|
if (r == 0)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
r--; /* make it zero based */
|
|
|
|
|
|
|
|
kscratch_used_mask |= (1 << r);
|
|
|
|
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static int scratch_reg;
|
|
|
|
static int pgd_reg;
|
2010-12-27 19:07:57 -07:00
|
|
|
enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch};
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static struct work_registers build_get_work_registers(u32 **p)
|
2011-07-05 17:34:46 -06:00
|
|
|
{
|
|
|
|
struct work_registers r;
|
|
|
|
|
2013-06-11 08:41:36 -06:00
|
|
|
if (scratch_reg >= 0) {
|
2011-07-05 17:34:46 -06:00
|
|
|
/* Save in CPU local C0_KScratch? */
|
2013-06-11 08:41:35 -06:00
|
|
|
UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg);
|
2011-07-05 17:34:46 -06:00
|
|
|
r.r1 = K0;
|
|
|
|
r.r2 = K1;
|
|
|
|
r.r3 = 1;
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (num_possible_cpus() > 1) {
|
|
|
|
/* Get smp_processor_id */
|
2013-08-11 05:40:16 -06:00
|
|
|
UASM_i_CPUID_MFC0(p, K0, SMP_CPUID_REG);
|
|
|
|
UASM_i_SRL_SAFE(p, K0, K0, SMP_CPUID_REGSHIFT);
|
2011-07-05 17:34:46 -06:00
|
|
|
|
|
|
|
/* handler_reg_save index in K0 */
|
|
|
|
UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save)));
|
|
|
|
|
|
|
|
UASM_i_LA(p, K1, (long)&handler_reg_save);
|
|
|
|
UASM_i_ADDU(p, K0, K0, K1);
|
|
|
|
} else {
|
|
|
|
UASM_i_LA(p, K0, (long)&handler_reg_save);
|
|
|
|
}
|
|
|
|
/* K0 now points to save area, save $1 and $2 */
|
|
|
|
UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0);
|
|
|
|
UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0);
|
|
|
|
|
|
|
|
r.r1 = K1;
|
|
|
|
r.r2 = 1;
|
|
|
|
r.r3 = 2;
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_restore_work_registers(u32 **p)
|
2011-07-05 17:34:46 -06:00
|
|
|
{
|
2013-06-11 08:41:36 -06:00
|
|
|
if (scratch_reg >= 0) {
|
2013-06-11 08:41:35 -06:00
|
|
|
UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
|
2011-07-05 17:34:46 -06:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
/* K0 already points to save area, restore $1 and $2 */
|
|
|
|
UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0);
|
|
|
|
UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0);
|
|
|
|
}
|
|
|
|
|
2010-12-27 19:07:57 -07:00
|
|
|
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
|
2010-12-21 15:19:11 -07:00
|
|
|
|
2009-10-14 13:16:56 -06:00
|
|
|
/*
|
|
|
|
* CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
|
|
|
|
* we cannot do r3000 under these circumstances.
|
2010-12-21 15:19:11 -07:00
|
|
|
*
|
|
|
|
* Declare pgd_current here instead of including mmu_context.h to avoid type
|
|
|
|
* conflicts for tlbmiss_handler_setup_pgd
|
2009-10-14 13:16:56 -06:00
|
|
|
*/
|
2010-12-21 15:19:11 -07:00
|
|
|
extern unsigned long pgd_current[];
|
2009-10-14 13:16:56 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* The R3000 TLB handler is simple.
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_r3000_tlb_refill_handler(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
long pgdc = (long)pgd_current;
|
|
|
|
u32 *p;
|
|
|
|
|
|
|
|
memset(tlb_handler, 0, sizeof(tlb_handler));
|
|
|
|
p = tlb_handler;
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_mfc0(&p, K0, C0_BADVADDR);
|
|
|
|
uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */
|
|
|
|
uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1);
|
|
|
|
uasm_i_srl(&p, K0, K0, 22); /* load delay */
|
|
|
|
uasm_i_sll(&p, K0, K0, 2);
|
|
|
|
uasm_i_addu(&p, K1, K1, K0);
|
|
|
|
uasm_i_mfc0(&p, K0, C0_CONTEXT);
|
|
|
|
uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */
|
|
|
|
uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */
|
|
|
|
uasm_i_addu(&p, K1, K1, K0);
|
|
|
|
uasm_i_lw(&p, K0, 0, K1);
|
|
|
|
uasm_i_nop(&p); /* load delay */
|
|
|
|
uasm_i_mtc0(&p, K0, C0_ENTRYLO0);
|
|
|
|
uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
|
|
|
|
uasm_i_tlbwr(&p); /* cp0 delay */
|
|
|
|
uasm_i_jr(&p, K1);
|
|
|
|
uasm_i_rfe(&p); /* branch delay */
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (p > tlb_handler + 32)
|
|
|
|
panic("TLB refill handler space exceeded");
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
pr_debug("Wrote TLB refill handler (%u instructions).\n",
|
|
|
|
(unsigned int)(p - tlb_handler));
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-03-29 10:53:00 -07:00
|
|
|
memcpy((void *)ebase, tlb_handler, 0x80);
|
2014-07-11 16:18:05 -06:00
|
|
|
local_flush_icache_range(ebase, ebase + 0x80);
|
2007-10-18 01:11:17 -06:00
|
|
|
|
2012-10-16 14:20:26 -06:00
|
|
|
dump_handler("r3000_tlb_refill", (u32 *)ebase, 32);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2009-10-14 13:16:56 -06:00
|
|
|
#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The R4000 TLB handler is much more complicated. We have two
|
|
|
|
* consecutive handler areas with 32 instructions space each.
|
|
|
|
* Since they aren't used at the same time, we can overflow in the
|
|
|
|
* other one.To keep things simple, we first assume linear space,
|
|
|
|
* then we relocate it to the final handler layout as needed.
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static u32 final_handler[64];
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Hazards
|
|
|
|
*
|
|
|
|
* From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0:
|
|
|
|
* 2. A timing hazard exists for the TLBP instruction.
|
|
|
|
*
|
2013-01-22 04:59:30 -07:00
|
|
|
* stalling_instruction
|
|
|
|
* TLBP
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* The JTLB is being read for the TLBP throughout the stall generated by the
|
|
|
|
* previous instruction. This is not really correct as the stalling instruction
|
|
|
|
* can modify the address used to access the JTLB. The failure symptom is that
|
|
|
|
* the TLBP instruction will use an address created for the stalling instruction
|
|
|
|
* and not the address held in C0_ENHI and thus report the wrong results.
|
|
|
|
*
|
|
|
|
* The software work-around is to not allow the instruction preceding the TLBP
|
|
|
|
* to stall - make it an NOP or some other instruction guaranteed not to stall.
|
|
|
|
*
|
2013-01-22 04:59:30 -07:00
|
|
|
* Errata 2 will not be fixed. This errata is also on the R5000.
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* As if we MIPS hackers wouldn't know how to nop pipelines happy ...
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void __maybe_unused build_tlb_probe_entry(u32 **p)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-10-11 16:46:15 -06:00
|
|
|
switch (current_cpu_type()) {
|
2008-05-12 05:55:42 -06:00
|
|
|
/* Found by experiment: R4600 v2.0/R4700 needs this, too. */
|
2005-09-09 11:11:50 -06:00
|
|
|
case CPU_R4600:
|
2008-05-12 05:55:42 -06:00
|
|
|
case CPU_R4700:
|
2005-04-16 16:20:36 -06:00
|
|
|
case CPU_R5000:
|
|
|
|
case CPU_NEVADA:
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(p);
|
|
|
|
uasm_i_tlbp(p);
|
2005-04-16 16:20:36 -06:00
|
|
|
break;
|
|
|
|
|
|
|
|
default:
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_tlbp(p);
|
2005-04-16 16:20:36 -06:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Write random or indexed TLB entry, and care about the hazards from
|
2011-03-30 19:57:33 -06:00
|
|
|
* the preceding mtc0 and for the following eret.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
enum tlb_write_entry { tlb_random, tlb_indexed };
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_tlb_write_entry(u32 **p, struct uasm_label **l,
|
|
|
|
struct uasm_reloc **r,
|
|
|
|
enum tlb_write_entry wmode)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
void(*tlbw)(u32 **) = NULL;
|
|
|
|
|
|
|
|
switch (wmode) {
|
2008-01-28 13:05:38 -07:00
|
|
|
case tlb_random: tlbw = uasm_i_tlbwr; break;
|
|
|
|
case tlb_indexed: tlbw = uasm_i_tlbwi; break;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2015-03-25 06:18:27 -06:00
|
|
|
if (cpu_has_mips_r2_r6) {
|
|
|
|
if (cpu_has_mips_r2_exec_hazard)
|
2009-05-12 13:41:53 -06:00
|
|
|
uasm_i_ehb(p);
|
2008-01-29 03:14:54 -07:00
|
|
|
tlbw(p);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2007-10-11 16:46:15 -06:00
|
|
|
switch (current_cpu_type()) {
|
2005-04-16 16:20:36 -06:00
|
|
|
case CPU_R4000PC:
|
|
|
|
case CPU_R4000SC:
|
|
|
|
case CPU_R4000MC:
|
|
|
|
case CPU_R4400PC:
|
|
|
|
case CPU_R4400SC:
|
|
|
|
case CPU_R4400MC:
|
|
|
|
/*
|
|
|
|
* This branch uses up a mtc0 hazard nop slot and saves
|
|
|
|
* two nops after the tlbw instruction.
|
|
|
|
*/
|
2012-10-13 14:46:26 -06:00
|
|
|
uasm_bgezl_hazard(p, r, hazard_instance);
|
2005-04-16 16:20:36 -06:00
|
|
|
tlbw(p);
|
2012-10-13 14:46:26 -06:00
|
|
|
uasm_bgezl_label(l, p, hazard_instance);
|
|
|
|
hazard_instance++;
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(p);
|
2005-04-16 16:20:36 -06:00
|
|
|
break;
|
|
|
|
|
|
|
|
case CPU_R4600:
|
|
|
|
case CPU_R4700:
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(p);
|
2005-06-30 04:51:01 -06:00
|
|
|
tlbw(p);
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(p);
|
2005-06-30 04:51:01 -06:00
|
|
|
break;
|
|
|
|
|
2012-10-16 14:13:06 -06:00
|
|
|
case CPU_R5000:
|
|
|
|
case CPU_NEVADA:
|
|
|
|
uasm_i_nop(p); /* QED specifies 2 nops hazard */
|
|
|
|
uasm_i_nop(p); /* QED specifies 2 nops hazard */
|
|
|
|
tlbw(p);
|
|
|
|
break;
|
|
|
|
|
2005-06-30 04:51:01 -06:00
|
|
|
case CPU_R4300:
|
2005-04-16 16:20:36 -06:00
|
|
|
case CPU_5KC:
|
|
|
|
case CPU_TX49XX:
|
2005-07-14 11:47:57 -06:00
|
|
|
case CPU_PR4450:
|
2011-05-06 14:06:21 -06:00
|
|
|
case CPU_XLR:
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(p);
|
2005-04-16 16:20:36 -06:00
|
|
|
tlbw(p);
|
|
|
|
break;
|
|
|
|
|
|
|
|
case CPU_R10000:
|
|
|
|
case CPU_R12000:
|
2006-05-16 20:23:59 -06:00
|
|
|
case CPU_R14000:
|
2015-01-21 05:59:45 -07:00
|
|
|
case CPU_R16000:
|
2005-04-16 16:20:36 -06:00
|
|
|
case CPU_4KC:
|
2008-03-26 09:42:54 -06:00
|
|
|
case CPU_4KEC:
|
2012-07-06 15:56:00 -06:00
|
|
|
case CPU_M14KC:
|
2012-12-06 20:51:35 -07:00
|
|
|
case CPU_M14KEC:
|
2005-04-16 16:20:36 -06:00
|
|
|
case CPU_SB1:
|
2005-10-20 00:56:20 -06:00
|
|
|
case CPU_SB1A:
|
2005-04-16 16:20:36 -06:00
|
|
|
case CPU_4KSC:
|
|
|
|
case CPU_20KC:
|
|
|
|
case CPU_25KF:
|
2010-10-16 15:22:30 -06:00
|
|
|
case CPU_BMIPS32:
|
|
|
|
case CPU_BMIPS3300:
|
|
|
|
case CPU_BMIPS4350:
|
|
|
|
case CPU_BMIPS4380:
|
|
|
|
case CPU_BMIPS5000:
|
2007-06-06 00:52:43 -06:00
|
|
|
case CPU_LOONGSON2:
|
2014-03-21 04:44:00 -06:00
|
|
|
case CPU_LOONGSON3:
|
2009-03-03 02:05:51 -07:00
|
|
|
case CPU_R5500:
|
2006-08-23 07:26:50 -06:00
|
|
|
if (m4kc_tlbp_war())
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(p);
|
2009-03-25 10:49:30 -06:00
|
|
|
case CPU_ALCHEMY:
|
2005-04-16 16:20:36 -06:00
|
|
|
tlbw(p);
|
|
|
|
break;
|
|
|
|
|
|
|
|
case CPU_RM7000:
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(p);
|
|
|
|
uasm_i_nop(p);
|
|
|
|
uasm_i_nop(p);
|
|
|
|
uasm_i_nop(p);
|
2005-04-16 16:20:36 -06:00
|
|
|
tlbw(p);
|
|
|
|
break;
|
|
|
|
|
|
|
|
case CPU_VR4111:
|
|
|
|
case CPU_VR4121:
|
|
|
|
case CPU_VR4122:
|
|
|
|
case CPU_VR4181:
|
|
|
|
case CPU_VR4181A:
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(p);
|
|
|
|
uasm_i_nop(p);
|
2005-04-16 16:20:36 -06:00
|
|
|
tlbw(p);
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(p);
|
|
|
|
uasm_i_nop(p);
|
2005-04-16 16:20:36 -06:00
|
|
|
break;
|
|
|
|
|
|
|
|
case CPU_VR4131:
|
|
|
|
case CPU_VR4133:
|
2005-08-29 10:49:55 -06:00
|
|
|
case CPU_R5432:
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(p);
|
|
|
|
uasm_i_nop(p);
|
2005-04-16 16:20:36 -06:00
|
|
|
tlbw(p);
|
|
|
|
break;
|
|
|
|
|
2010-07-17 05:07:51 -06:00
|
|
|
case CPU_JZRISC:
|
|
|
|
tlbw(p);
|
|
|
|
uasm_i_nop(p);
|
|
|
|
break;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
default:
|
|
|
|
panic("No TLB refill handler yet (CPU type: %d)",
|
2010-12-25 13:42:37 -07:00
|
|
|
current_cpu_type());
|
2005-04-16 16:20:36 -06:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
|
|
|
|
unsigned int reg)
|
2009-05-27 18:47:44 -06:00
|
|
|
{
|
2016-04-19 02:25:09 -06:00
|
|
|
if (_PAGE_GLOBAL_SHIFT == 0) {
|
|
|
|
/* pte_t is already in EntryLo format */
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2015-09-22 12:42:52 -06:00
|
|
|
if (cpu_has_rixi && _PAGE_NO_EXEC) {
|
|
|
|
if (fill_includes_sw_bits) {
|
|
|
|
UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL));
|
|
|
|
} else {
|
|
|
|
UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC));
|
|
|
|
UASM_i_ROTR(p, reg, reg,
|
|
|
|
ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
|
|
|
|
}
|
2010-02-10 16:12:47 -07:00
|
|
|
} else {
|
2014-11-21 16:16:48 -07:00
|
|
|
#ifdef CONFIG_PHYS_ADDR_T_64BIT
|
2010-04-28 13:16:17 -06:00
|
|
|
uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL));
|
2010-02-10 16:12:47 -07:00
|
|
|
#else
|
|
|
|
UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL));
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
}
|
2009-05-27 18:47:44 -06:00
|
|
|
|
2012-10-16 16:48:10 -06:00
|
|
|
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
|
2009-05-27 18:47:44 -06:00
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_restore_pagemask(u32 **p, struct uasm_reloc **r,
|
|
|
|
unsigned int tmp, enum label_id lid,
|
|
|
|
int restore_scratch)
|
2010-02-10 16:12:47 -07:00
|
|
|
{
|
2010-12-27 19:07:57 -07:00
|
|
|
if (restore_scratch) {
|
|
|
|
/* Reset default page size */
|
|
|
|
if (PM_DEFAULT_MASK >> 16) {
|
|
|
|
uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
|
|
|
|
uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
|
|
|
|
uasm_i_mtc0(p, tmp, C0_PAGEMASK);
|
|
|
|
uasm_il_b(p, r, lid);
|
|
|
|
} else if (PM_DEFAULT_MASK) {
|
|
|
|
uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
|
|
|
|
uasm_i_mtc0(p, tmp, C0_PAGEMASK);
|
|
|
|
uasm_il_b(p, r, lid);
|
|
|
|
} else {
|
|
|
|
uasm_i_mtc0(p, 0, C0_PAGEMASK);
|
|
|
|
uasm_il_b(p, r, lid);
|
|
|
|
}
|
2013-06-11 08:41:36 -06:00
|
|
|
if (scratch_reg >= 0)
|
2013-06-11 08:41:35 -06:00
|
|
|
UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
|
2010-12-27 19:07:57 -07:00
|
|
|
else
|
|
|
|
UASM_i_LW(p, 1, scratchpad_offset(0), 0);
|
2009-05-27 18:47:44 -06:00
|
|
|
} else {
|
2010-12-27 19:07:57 -07:00
|
|
|
/* Reset default page size */
|
|
|
|
if (PM_DEFAULT_MASK >> 16) {
|
|
|
|
uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
|
|
|
|
uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
|
|
|
|
uasm_il_b(p, r, lid);
|
|
|
|
uasm_i_mtc0(p, tmp, C0_PAGEMASK);
|
|
|
|
} else if (PM_DEFAULT_MASK) {
|
|
|
|
uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
|
|
|
|
uasm_il_b(p, r, lid);
|
|
|
|
uasm_i_mtc0(p, tmp, C0_PAGEMASK);
|
|
|
|
} else {
|
|
|
|
uasm_il_b(p, r, lid);
|
|
|
|
uasm_i_mtc0(p, 0, C0_PAGEMASK);
|
|
|
|
}
|
2009-05-27 18:47:44 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l,
|
|
|
|
struct uasm_reloc **r,
|
|
|
|
unsigned int tmp,
|
|
|
|
enum tlb_write_entry wmode,
|
|
|
|
int restore_scratch)
|
2010-02-10 16:12:47 -07:00
|
|
|
{
|
|
|
|
/* Set huge page tlb entry size */
|
|
|
|
uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
|
|
|
|
uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff);
|
|
|
|
uasm_i_mtc0(p, tmp, C0_PAGEMASK);
|
|
|
|
|
|
|
|
build_tlb_write_entry(p, l, r, wmode);
|
|
|
|
|
2010-12-27 19:07:57 -07:00
|
|
|
build_restore_pagemask(p, r, tmp, label_leave, restore_scratch);
|
2010-02-10 16:12:47 -07:00
|
|
|
}
|
|
|
|
|
2009-05-27 18:47:44 -06:00
|
|
|
/*
|
|
|
|
* Check if Huge PTE is present, if so then jump to LABEL.
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void
|
2009-05-27 18:47:44 -06:00
|
|
|
build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
unsigned int pmd, int lid)
|
2009-05-27 18:47:44 -06:00
|
|
|
{
|
|
|
|
UASM_i_LW(p, tmp, 0, pmd);
|
2010-12-20 16:54:50 -07:00
|
|
|
if (use_bbit_insns()) {
|
|
|
|
uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid);
|
|
|
|
} else {
|
|
|
|
uasm_i_andi(p, tmp, tmp, _PAGE_HUGE);
|
|
|
|
uasm_il_bnez(p, r, tmp, lid);
|
|
|
|
}
|
2009-05-27 18:47:44 -06:00
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_huge_update_entries(u32 **p, unsigned int pte,
|
|
|
|
unsigned int tmp)
|
2009-05-27 18:47:44 -06:00
|
|
|
{
|
|
|
|
int small_sequence;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* A huge PTE describes an area the size of the
|
|
|
|
* configured huge page size. This is twice the
|
|
|
|
* of the large TLB entry size we intend to use.
|
|
|
|
* A TLB entry half the size of the configured
|
|
|
|
* huge page size is configured into entrylo0
|
|
|
|
* and entrylo1 to cover the contiguous huge PTE
|
|
|
|
* address space.
|
|
|
|
*/
|
|
|
|
small_sequence = (HPAGE_SIZE >> 7) < 0x10000;
|
|
|
|
|
2013-01-22 04:59:30 -07:00
|
|
|
/* We can clobber tmp. It isn't used after this.*/
|
2009-05-27 18:47:44 -06:00
|
|
|
if (!small_sequence)
|
|
|
|
uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16));
|
|
|
|
|
2010-02-10 16:12:47 -07:00
|
|
|
build_convert_pte_to_entrylo(p, pte);
|
2010-02-10 16:12:44 -07:00
|
|
|
UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */
|
2009-05-27 18:47:44 -06:00
|
|
|
/* convert to entrylo1 */
|
|
|
|
if (small_sequence)
|
|
|
|
UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7);
|
|
|
|
else
|
|
|
|
UASM_i_ADDU(p, pte, pte, tmp);
|
|
|
|
|
2010-02-10 16:12:44 -07:00
|
|
|
UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */
|
2009-05-27 18:47:44 -06:00
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r,
|
|
|
|
struct uasm_label **l,
|
|
|
|
unsigned int pte,
|
|
|
|
unsigned int ptr)
|
2009-05-27 18:47:44 -06:00
|
|
|
{
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
UASM_i_SC(p, pte, 0, ptr);
|
|
|
|
uasm_il_beqz(p, r, pte, label_tlb_huge_update);
|
|
|
|
UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */
|
|
|
|
#else
|
|
|
|
UASM_i_SW(p, pte, 0, ptr);
|
|
|
|
#endif
|
|
|
|
build_huge_update_entries(p, pte, ptr);
|
2010-12-27 19:07:57 -07:00
|
|
|
build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0);
|
2009-05-27 18:47:44 -06:00
|
|
|
}
|
2012-10-16 16:48:10 -06:00
|
|
|
#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */
|
2009-05-27 18:47:44 -06:00
|
|
|
|
2005-09-03 16:56:16 -06:00
|
|
|
#ifdef CONFIG_64BIT
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* TMP and PTR are scratch.
|
|
|
|
* TMP will be clobbered, PTR will hold the pmd entry.
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void
|
2008-01-28 13:05:38 -07:00
|
|
|
build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
|
2005-04-16 16:20:36 -06:00
|
|
|
unsigned int tmp, unsigned int ptr)
|
|
|
|
{
|
2009-10-14 13:16:56 -06:00
|
|
|
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
|
2005-04-16 16:20:36 -06:00
|
|
|
long pgdc = (long)pgd_current;
|
2009-10-14 13:16:56 -06:00
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* The vmalloc handling is not in the hotpath.
|
|
|
|
*/
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_dmfc0(p, tmp, C0_BADVADDR);
|
2010-04-28 13:16:18 -06:00
|
|
|
|
|
|
|
if (check_for_high_segbits) {
|
|
|
|
/*
|
|
|
|
* The kernel currently implicitely assumes that the
|
|
|
|
* MIPS SEGBITS parameter for the processor is
|
|
|
|
* (PGDIR_SHIFT+PGDIR_BITS) or less, and will never
|
|
|
|
* allocate virtual addresses outside the maximum
|
|
|
|
* range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But
|
|
|
|
* that doesn't prevent user code from accessing the
|
|
|
|
* higher xuseg addresses. Here, we make sure that
|
|
|
|
* everything but the lower xuseg addresses goes down
|
|
|
|
* the module_alloc/vmalloc path.
|
|
|
|
*/
|
|
|
|
uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
|
|
|
|
uasm_il_bnez(p, r, ptr, label_vmalloc);
|
|
|
|
} else {
|
|
|
|
uasm_il_bltz(p, r, tmp, label_vmalloc);
|
|
|
|
}
|
2008-01-28 13:05:38 -07:00
|
|
|
/* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2010-12-21 15:19:11 -07:00
|
|
|
if (pgd_reg != -1) {
|
|
|
|
/* pgd is in pgd_reg */
|
2016-03-02 18:45:12 -07:00
|
|
|
if (cpu_has_ldpte)
|
|
|
|
UASM_i_MFC0(p, ptr, C0_PWBASE);
|
|
|
|
else
|
|
|
|
UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
|
2010-12-21 15:19:11 -07:00
|
|
|
} else {
|
2013-09-25 04:58:04 -06:00
|
|
|
#if defined(CONFIG_MIPS_PGD_C0_CONTEXT)
|
2010-12-21 15:19:11 -07:00
|
|
|
/*
|
|
|
|
* &pgd << 11 stored in CONTEXT [23..63].
|
|
|
|
*/
|
|
|
|
UASM_i_MFC0(p, ptr, C0_CONTEXT);
|
|
|
|
|
|
|
|
/* Clear lower 23 bits of context. */
|
|
|
|
uasm_i_dins(p, ptr, 0, 0, 23);
|
|
|
|
|
2013-01-22 04:59:30 -07:00
|
|
|
/* 1 0 1 0 1 << 6 xkphys cached */
|
2010-12-21 15:19:11 -07:00
|
|
|
uasm_i_ori(p, ptr, ptr, 0x540);
|
|
|
|
uasm_i_drotr(p, ptr, ptr, 11);
|
2009-10-14 13:16:56 -06:00
|
|
|
#elif defined(CONFIG_SMP)
|
2013-09-25 04:58:04 -06:00
|
|
|
UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG);
|
|
|
|
uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT);
|
|
|
|
UASM_i_LA_mostly(p, tmp, pgdc);
|
|
|
|
uasm_i_daddu(p, ptr, ptr, tmp);
|
|
|
|
uasm_i_dmfc0(p, tmp, C0_BADVADDR);
|
|
|
|
uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
#else
|
2013-09-25 04:58:04 -06:00
|
|
|
UASM_i_LA_mostly(p, ptr, pgdc);
|
|
|
|
uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
2013-09-25 04:58:04 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_l_vmalloc_done(l, *p);
|
2006-10-23 19:29:01 -06:00
|
|
|
|
2010-04-28 13:16:17 -06:00
|
|
|
/* get pgd offset in bytes */
|
|
|
|
uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3);
|
2008-01-28 13:05:38 -07:00
|
|
|
|
|
|
|
uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
|
|
|
|
uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
|
2009-12-04 14:52:36 -07:00
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
|
|
|
|
uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */
|
2010-04-28 13:16:17 -06:00
|
|
|
uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
|
|
|
|
uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
|
2009-12-04 14:52:36 -07:00
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* BVADDR is the faulting address, PTR is scratch.
|
|
|
|
* PTR will hold the pgd for vmalloc.
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void
|
2008-01-28 13:05:38 -07:00
|
|
|
build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
|
2010-04-28 13:16:18 -06:00
|
|
|
unsigned int bvaddr, unsigned int ptr,
|
|
|
|
enum vmalloc64_mode mode)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
long swpd = (long)swapper_pg_dir;
|
2010-04-28 13:16:18 -06:00
|
|
|
int single_insn_swpd;
|
|
|
|
int did_vmalloc_branch = 0;
|
|
|
|
|
|
|
|
single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_l_vmalloc(l, *p);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2010-12-27 19:07:57 -07:00
|
|
|
if (mode != not_refill && check_for_high_segbits) {
|
2010-04-28 13:16:18 -06:00
|
|
|
if (single_insn_swpd) {
|
|
|
|
uasm_il_bltz(p, r, bvaddr, label_vmalloc_done);
|
|
|
|
uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
|
|
|
|
did_vmalloc_branch = 1;
|
|
|
|
/* fall through */
|
|
|
|
} else {
|
|
|
|
uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!did_vmalloc_branch) {
|
|
|
|
if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
|
|
|
|
uasm_il_b(p, r, label_vmalloc_done);
|
|
|
|
uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
|
|
|
|
} else {
|
|
|
|
UASM_i_LA_mostly(p, ptr, swpd);
|
|
|
|
uasm_il_b(p, r, label_vmalloc_done);
|
|
|
|
if (uasm_in_compat_space_p(swpd))
|
|
|
|
uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
|
|
|
|
else
|
|
|
|
uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
|
|
|
|
}
|
|
|
|
}
|
2010-12-27 19:07:57 -07:00
|
|
|
if (mode != not_refill && check_for_high_segbits) {
|
2010-04-28 13:16:18 -06:00
|
|
|
uasm_l_large_segbits_fault(l, *p);
|
|
|
|
/*
|
|
|
|
* We get here if we are an xsseg address, or if we are
|
|
|
|
* an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary.
|
|
|
|
*
|
|
|
|
* Ignoring xsseg (assume disabled so would generate
|
|
|
|
* (address errors?), the only remaining possibility
|
|
|
|
* is the upper xuseg addresses. On processors with
|
|
|
|
* TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these
|
|
|
|
* addresses would have taken an address error. We try
|
|
|
|
* to mimic that here by taking a load/istream page
|
|
|
|
* fault.
|
|
|
|
*/
|
|
|
|
UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0);
|
|
|
|
uasm_i_jr(p, ptr);
|
2010-12-27 19:07:57 -07:00
|
|
|
|
|
|
|
if (mode == refill_scratch) {
|
2013-06-11 08:41:36 -06:00
|
|
|
if (scratch_reg >= 0)
|
2013-06-11 08:41:35 -06:00
|
|
|
UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
|
2010-12-27 19:07:57 -07:00
|
|
|
else
|
|
|
|
UASM_i_LW(p, 1, scratchpad_offset(0), 0);
|
|
|
|
} else {
|
|
|
|
uasm_i_nop(p);
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2005-09-03 16:56:16 -06:00
|
|
|
#else /* !CONFIG_64BIT */
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* TMP and PTR are scratch.
|
|
|
|
* TMP will be clobbered, PTR will hold the pgd entry.
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void __maybe_unused
|
2005-04-16 16:20:36 -06:00
|
|
|
build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
|
|
|
|
{
|
2013-09-25 04:58:04 -06:00
|
|
|
if (pgd_reg != -1) {
|
|
|
|
/* pgd is in pgd_reg */
|
|
|
|
uasm_i_mfc0(p, ptr, c0_kscratch(), pgd_reg);
|
|
|
|
uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
|
|
|
|
} else {
|
|
|
|
long pgdc = (long)pgd_current;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-09-25 04:58:04 -06:00
|
|
|
/* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
|
2005-04-16 16:20:36 -06:00
|
|
|
#ifdef CONFIG_SMP
|
2013-09-25 04:58:04 -06:00
|
|
|
uasm_i_mfc0(p, ptr, SMP_CPUID_REG);
|
|
|
|
UASM_i_LA_mostly(p, tmp, pgdc);
|
|
|
|
uasm_i_srl(p, ptr, ptr, SMP_CPUID_PTRSHIFT);
|
|
|
|
uasm_i_addu(p, ptr, tmp, ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
#else
|
2013-09-25 04:58:04 -06:00
|
|
|
UASM_i_LA_mostly(p, ptr, pgdc);
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
2013-09-25 04:58:04 -06:00
|
|
|
uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
|
|
|
|
uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
|
|
|
|
}
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
|
|
|
|
uasm_i_sll(p, tmp, tmp, PGD_T_LOG2);
|
|
|
|
uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2005-09-03 16:56:16 -06:00
|
|
|
#endif /* !CONFIG_64BIT */
|
2005-04-16 16:20:36 -06:00
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_adjust_context(u32 **p, unsigned int ctx)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-10-23 19:29:01 -06:00
|
|
|
unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
|
2005-04-16 16:20:36 -06:00
|
|
|
unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);
|
|
|
|
|
2007-10-11 16:46:15 -06:00
|
|
|
switch (current_cpu_type()) {
|
2005-04-16 16:20:36 -06:00
|
|
|
case CPU_VR41XX:
|
|
|
|
case CPU_VR4111:
|
|
|
|
case CPU_VR4121:
|
|
|
|
case CPU_VR4122:
|
|
|
|
case CPU_VR4131:
|
|
|
|
case CPU_VR4181:
|
|
|
|
case CPU_VR4181A:
|
|
|
|
case CPU_VR4133:
|
|
|
|
shift += 2;
|
|
|
|
break;
|
|
|
|
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (shift)
|
2008-01-28 13:05:38 -07:00
|
|
|
UASM_i_SRL(p, ctx, ctx, shift);
|
|
|
|
uasm_i_andi(p, ctx, ctx, mask);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Bug workaround for the Nevada. It seems as if under certain
|
|
|
|
* circumstances the move from cp0_context might produce a
|
|
|
|
* bogus result when the mfc0 instruction and its consumer are
|
|
|
|
* in a different cacheline or a load instruction, probably any
|
|
|
|
* memory reference, is between them.
|
|
|
|
*/
|
2007-10-11 16:46:15 -06:00
|
|
|
switch (current_cpu_type()) {
|
2005-04-16 16:20:36 -06:00
|
|
|
case CPU_NEVADA:
|
2008-01-28 13:05:38 -07:00
|
|
|
UASM_i_LW(p, ptr, 0, ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
GET_CONTEXT(p, tmp); /* get context reg */
|
|
|
|
break;
|
|
|
|
|
|
|
|
default:
|
|
|
|
GET_CONTEXT(p, tmp); /* get context reg */
|
2008-01-28 13:05:38 -07:00
|
|
|
UASM_i_LW(p, ptr, 0, ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
build_adjust_context(p, tmp);
|
2008-01-28 13:05:38 -07:00
|
|
|
UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2016-04-19 02:25:09 -06:00
|
|
|
int pte_off_even = 0;
|
|
|
|
int pte_off_odd = sizeof(pte_t);
|
2016-04-19 02:25:05 -06:00
|
|
|
|
2016-04-19 02:25:09 -06:00
|
|
|
#if defined(CONFIG_CPU_MIPS32) && defined(CONFIG_PHYS_ADDR_T_64BIT)
|
|
|
|
/* The low 32 bits of EntryLo is stored in pte_high */
|
|
|
|
pte_off_even += offsetof(pte_t, pte_high);
|
|
|
|
pte_off_odd += offsetof(pte_t, pte_high);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
if (config_enabled(CONFIG_XPA)) {
|
2015-02-26 17:16:38 -07:00
|
|
|
uasm_i_lw(p, tmp, pte_off_even, ptep); /* even pte */
|
|
|
|
UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL));
|
|
|
|
UASM_i_MTC0(p, tmp, C0_ENTRYLO0);
|
2016-04-19 02:25:05 -06:00
|
|
|
|
2016-04-19 02:25:10 -06:00
|
|
|
if (cpu_has_xpa && !mips_xpa_disabled) {
|
|
|
|
uasm_i_lw(p, tmp, 0, ptep);
|
|
|
|
uasm_i_ext(p, tmp, tmp, 0, 24);
|
|
|
|
uasm_i_mthc0(p, tmp, C0_ENTRYLO0);
|
|
|
|
}
|
2016-04-19 02:25:06 -06:00
|
|
|
|
|
|
|
uasm_i_lw(p, tmp, pte_off_odd, ptep); /* odd pte */
|
|
|
|
UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL));
|
|
|
|
UASM_i_MTC0(p, tmp, C0_ENTRYLO1);
|
|
|
|
|
2016-04-19 02:25:10 -06:00
|
|
|
if (cpu_has_xpa && !mips_xpa_disabled) {
|
|
|
|
uasm_i_lw(p, tmp, sizeof(pte_t), ptep);
|
|
|
|
uasm_i_ext(p, tmp, tmp, 0, 24);
|
|
|
|
uasm_i_mthc0(p, tmp, C0_ENTRYLO1);
|
|
|
|
}
|
2016-04-19 02:25:05 -06:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2016-04-19 02:25:09 -06:00
|
|
|
UASM_i_LW(p, tmp, pte_off_even, ptep); /* get even pte */
|
|
|
|
UASM_i_LW(p, ptep, pte_off_odd, ptep); /* get odd pte */
|
2005-04-16 16:20:36 -06:00
|
|
|
if (r45k_bvahwbug())
|
|
|
|
build_tlb_probe_entry(p);
|
2015-09-22 12:42:49 -06:00
|
|
|
build_convert_pte_to_entrylo(p, tmp);
|
|
|
|
if (r4k_250MHZhwbug())
|
|
|
|
UASM_i_MTC0(p, 0, C0_ENTRYLO0);
|
|
|
|
UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
|
|
|
|
build_convert_pte_to_entrylo(p, ptep);
|
|
|
|
if (r45k_bvahwbug())
|
|
|
|
uasm_i_mfc0(p, tmp, C0_INDEX);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (r4k_250MHZhwbug())
|
2010-02-10 16:12:44 -07:00
|
|
|
UASM_i_MTC0(p, 0, C0_ENTRYLO1);
|
|
|
|
UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2010-12-27 19:07:57 -07:00
|
|
|
struct mips_huge_tlb_info {
|
|
|
|
int huge_pte;
|
|
|
|
int restore_scratch;
|
MIPS: tlbex: Properly fix HUGE TLB Refill exception handler
In commit 8393c524a25609 (MIPS: tlbex: Fix a missing statement for
HUGETLB), the TLB Refill handler was fixed so that non-OCTEON targets
would work properly with huge pages. The change was incorrect in that
it broke the OCTEON case.
The problem is shown here:
xxx0: df7a0000 ld k0,0(k1)
.
.
.
xxxc0: df610000 ld at,0(k1)
xxxc4: 335a0ff0 andi k0,k0,0xff0
xxxc8: e825ffcd bbit1 at,0x5,0x0
xxxcc: 003ad82d daddu k1,at,k0
.
.
.
In the non-octeon case there is a destructive test for the huge PTE
bit, and then at 0, $k0 is reloaded (that is what the 8393c524a25609
patch added).
In the octeon case, we modify k1 in the branch delay slot, but we
never need k0 again, so the new load is not needed, but since k1 is
modified, if we do the load, we load from a garbage location and then
get a nested TLB Refill, which is seen in userspace as either SIGBUS
or SIGSEGV (depending on the garbage).
The real fix is to only do this reloading if it is needed, and never
where it is harmful.
Signed-off-by: David Daney <david.daney@cavium.com>
Cc: Huacai Chen <chenhc@lemote.com>
Cc: Fuxin Zhang <zhangfx@lemote.com>
Cc: Zhangjin Wu <wuzhangjin@gmail.com>
Cc: stable@vger.kernel.org
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/8151/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2014-10-20 16:34:23 -06:00
|
|
|
bool need_reload_pte;
|
2010-12-27 19:07:57 -07:00
|
|
|
};
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static struct mips_huge_tlb_info
|
2010-12-27 19:07:57 -07:00
|
|
|
build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
|
|
|
|
struct uasm_reloc **r, unsigned int tmp,
|
2013-06-11 08:41:35 -06:00
|
|
|
unsigned int ptr, int c0_scratch_reg)
|
2010-12-27 19:07:57 -07:00
|
|
|
{
|
|
|
|
struct mips_huge_tlb_info rv;
|
|
|
|
unsigned int even, odd;
|
|
|
|
int vmalloc_branch_delay_filled = 0;
|
|
|
|
const int scratch = 1; /* Our extra working register */
|
|
|
|
|
|
|
|
rv.huge_pte = scratch;
|
|
|
|
rv.restore_scratch = 0;
|
MIPS: tlbex: Properly fix HUGE TLB Refill exception handler
In commit 8393c524a25609 (MIPS: tlbex: Fix a missing statement for
HUGETLB), the TLB Refill handler was fixed so that non-OCTEON targets
would work properly with huge pages. The change was incorrect in that
it broke the OCTEON case.
The problem is shown here:
xxx0: df7a0000 ld k0,0(k1)
.
.
.
xxxc0: df610000 ld at,0(k1)
xxxc4: 335a0ff0 andi k0,k0,0xff0
xxxc8: e825ffcd bbit1 at,0x5,0x0
xxxcc: 003ad82d daddu k1,at,k0
.
.
.
In the non-octeon case there is a destructive test for the huge PTE
bit, and then at 0, $k0 is reloaded (that is what the 8393c524a25609
patch added).
In the octeon case, we modify k1 in the branch delay slot, but we
never need k0 again, so the new load is not needed, but since k1 is
modified, if we do the load, we load from a garbage location and then
get a nested TLB Refill, which is seen in userspace as either SIGBUS
or SIGSEGV (depending on the garbage).
The real fix is to only do this reloading if it is needed, and never
where it is harmful.
Signed-off-by: David Daney <david.daney@cavium.com>
Cc: Huacai Chen <chenhc@lemote.com>
Cc: Fuxin Zhang <zhangfx@lemote.com>
Cc: Zhangjin Wu <wuzhangjin@gmail.com>
Cc: stable@vger.kernel.org
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/8151/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2014-10-20 16:34:23 -06:00
|
|
|
rv.need_reload_pte = false;
|
2010-12-27 19:07:57 -07:00
|
|
|
|
|
|
|
if (check_for_high_segbits) {
|
|
|
|
UASM_i_MFC0(p, tmp, C0_BADVADDR);
|
|
|
|
|
|
|
|
if (pgd_reg != -1)
|
2013-06-11 08:41:35 -06:00
|
|
|
UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
|
2010-12-27 19:07:57 -07:00
|
|
|
else
|
|
|
|
UASM_i_MFC0(p, ptr, C0_CONTEXT);
|
|
|
|
|
2013-06-11 08:41:35 -06:00
|
|
|
if (c0_scratch_reg >= 0)
|
|
|
|
UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg);
|
2010-12-27 19:07:57 -07:00
|
|
|
else
|
|
|
|
UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
|
|
|
|
|
|
|
|
uasm_i_dsrl_safe(p, scratch, tmp,
|
|
|
|
PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
|
|
|
|
uasm_il_bnez(p, r, scratch, label_vmalloc);
|
|
|
|
|
|
|
|
if (pgd_reg == -1) {
|
|
|
|
vmalloc_branch_delay_filled = 1;
|
|
|
|
/* Clear lower 23 bits of context. */
|
|
|
|
uasm_i_dins(p, ptr, 0, 0, 23);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (pgd_reg != -1)
|
2013-06-11 08:41:35 -06:00
|
|
|
UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
|
2010-12-27 19:07:57 -07:00
|
|
|
else
|
|
|
|
UASM_i_MFC0(p, ptr, C0_CONTEXT);
|
|
|
|
|
|
|
|
UASM_i_MFC0(p, tmp, C0_BADVADDR);
|
|
|
|
|
2013-06-11 08:41:35 -06:00
|
|
|
if (c0_scratch_reg >= 0)
|
|
|
|
UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg);
|
2010-12-27 19:07:57 -07:00
|
|
|
else
|
|
|
|
UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
|
|
|
|
|
|
|
|
if (pgd_reg == -1)
|
|
|
|
/* Clear lower 23 bits of context. */
|
|
|
|
uasm_i_dins(p, ptr, 0, 0, 23);
|
|
|
|
|
|
|
|
uasm_il_bltz(p, r, tmp, label_vmalloc);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (pgd_reg == -1) {
|
|
|
|
vmalloc_branch_delay_filled = 1;
|
2013-01-22 04:59:30 -07:00
|
|
|
/* 1 0 1 0 1 << 6 xkphys cached */
|
2010-12-27 19:07:57 -07:00
|
|
|
uasm_i_ori(p, ptr, ptr, 0x540);
|
|
|
|
uasm_i_drotr(p, ptr, ptr, 11);
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef __PAGETABLE_PMD_FOLDED
|
|
|
|
#define LOC_PTEP scratch
|
|
|
|
#else
|
|
|
|
#define LOC_PTEP ptr
|
|
|
|
#endif
|
|
|
|
|
|
|
|
if (!vmalloc_branch_delay_filled)
|
|
|
|
/* get pgd offset in bytes */
|
|
|
|
uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
|
|
|
|
|
|
|
|
uasm_l_vmalloc_done(l, *p);
|
|
|
|
|
|
|
|
/*
|
2013-01-22 04:59:30 -07:00
|
|
|
* tmp ptr
|
|
|
|
* fall-through case = badvaddr *pgd_current
|
|
|
|
* vmalloc case = badvaddr swapper_pg_dir
|
2010-12-27 19:07:57 -07:00
|
|
|
*/
|
|
|
|
|
|
|
|
if (vmalloc_branch_delay_filled)
|
|
|
|
/* get pgd offset in bytes */
|
|
|
|
uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
|
|
|
|
|
|
|
|
#ifdef __PAGETABLE_PMD_FOLDED
|
|
|
|
GET_CONTEXT(p, tmp); /* get context reg */
|
|
|
|
#endif
|
|
|
|
uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3);
|
|
|
|
|
|
|
|
if (use_lwx_insns()) {
|
|
|
|
UASM_i_LWX(p, LOC_PTEP, scratch, ptr);
|
|
|
|
} else {
|
|
|
|
uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */
|
|
|
|
uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
|
|
|
/* get pmd offset in bytes */
|
|
|
|
uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3);
|
|
|
|
uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3);
|
|
|
|
GET_CONTEXT(p, tmp); /* get context reg */
|
|
|
|
|
|
|
|
if (use_lwx_insns()) {
|
|
|
|
UASM_i_LWX(p, scratch, scratch, ptr);
|
|
|
|
} else {
|
|
|
|
uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */
|
|
|
|
UASM_i_LW(p, scratch, 0, ptr);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
/* Adjust the context during the load latency. */
|
|
|
|
build_adjust_context(p, tmp);
|
|
|
|
|
2012-10-16 16:48:10 -06:00
|
|
|
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
|
2010-12-27 19:07:57 -07:00
|
|
|
uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update);
|
|
|
|
/*
|
|
|
|
* The in the LWX case we don't want to do the load in the
|
2013-01-22 04:59:30 -07:00
|
|
|
* delay slot. It cannot issue in the same cycle and may be
|
2010-12-27 19:07:57 -07:00
|
|
|
* speculative and unneeded.
|
|
|
|
*/
|
|
|
|
if (use_lwx_insns())
|
|
|
|
uasm_i_nop(p);
|
2012-10-16 16:48:10 -06:00
|
|
|
#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */
|
2010-12-27 19:07:57 -07:00
|
|
|
|
|
|
|
|
|
|
|
/* build_update_entries */
|
|
|
|
if (use_lwx_insns()) {
|
|
|
|
even = ptr;
|
|
|
|
odd = tmp;
|
|
|
|
UASM_i_LWX(p, even, scratch, tmp);
|
|
|
|
UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t));
|
|
|
|
UASM_i_LWX(p, odd, scratch, tmp);
|
|
|
|
} else {
|
|
|
|
UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */
|
|
|
|
even = tmp;
|
|
|
|
odd = ptr;
|
|
|
|
UASM_i_LW(p, even, 0, ptr); /* get even pte */
|
|
|
|
UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */
|
|
|
|
}
|
2012-09-13 15:51:46 -06:00
|
|
|
if (cpu_has_rixi) {
|
2012-08-23 11:02:03 -06:00
|
|
|
uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL));
|
2010-12-27 19:07:57 -07:00
|
|
|
UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
|
2012-08-23 11:02:03 -06:00
|
|
|
uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL));
|
2010-12-27 19:07:57 -07:00
|
|
|
} else {
|
|
|
|
uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL));
|
|
|
|
UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
|
|
|
|
uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL));
|
|
|
|
}
|
|
|
|
UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */
|
|
|
|
|
2013-06-11 08:41:35 -06:00
|
|
|
if (c0_scratch_reg >= 0) {
|
|
|
|
UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg);
|
2010-12-27 19:07:57 -07:00
|
|
|
build_tlb_write_entry(p, l, r, tlb_random);
|
|
|
|
uasm_l_leave(l, *p);
|
|
|
|
rv.restore_scratch = 1;
|
|
|
|
} else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) {
|
|
|
|
build_tlb_write_entry(p, l, r, tlb_random);
|
|
|
|
uasm_l_leave(l, *p);
|
|
|
|
UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
|
|
|
|
} else {
|
|
|
|
UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
|
|
|
|
build_tlb_write_entry(p, l, r, tlb_random);
|
|
|
|
uasm_l_leave(l, *p);
|
|
|
|
rv.restore_scratch = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
uasm_i_eret(p); /* return from trap */
|
|
|
|
|
|
|
|
return rv;
|
|
|
|
}
|
|
|
|
|
2009-05-20 12:40:58 -06:00
|
|
|
/*
|
|
|
|
* For a 64-bit kernel, we are using the 64-bit XTLB refill exception
|
|
|
|
* because EXL == 0. If we wrap, we can also use the 32 instruction
|
|
|
|
* slots before the XTLB refill exception handler which belong to the
|
|
|
|
* unused TLB refill exception.
|
|
|
|
*/
|
|
|
|
#define MIPS64_REFILL_INSNS 32
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_r4000_tlb_refill_handler(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
u32 *p = tlb_handler;
|
2008-01-28 13:05:38 -07:00
|
|
|
struct uasm_label *l = labels;
|
|
|
|
struct uasm_reloc *r = relocs;
|
2005-04-16 16:20:36 -06:00
|
|
|
u32 *f;
|
|
|
|
unsigned int final_len;
|
2011-03-29 02:54:54 -06:00
|
|
|
struct mips_huge_tlb_info htlb_info __maybe_unused;
|
|
|
|
enum vmalloc64_mode vmalloc_mode __maybe_unused;
|
2014-05-28 15:52:13 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
memset(tlb_handler, 0, sizeof(tlb_handler));
|
|
|
|
memset(labels, 0, sizeof(labels));
|
|
|
|
memset(relocs, 0, sizeof(relocs));
|
|
|
|
memset(final_handler, 0, sizeof(final_handler));
|
|
|
|
|
2014-05-28 15:52:13 -06:00
|
|
|
if (IS_ENABLED(CONFIG_64BIT) && (scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) {
|
2010-12-27 19:07:57 -07:00
|
|
|
htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1,
|
|
|
|
scratch_reg);
|
|
|
|
vmalloc_mode = refill_scratch;
|
|
|
|
} else {
|
|
|
|
htlb_info.huge_pte = K0;
|
|
|
|
htlb_info.restore_scratch = 0;
|
MIPS: tlbex: Properly fix HUGE TLB Refill exception handler
In commit 8393c524a25609 (MIPS: tlbex: Fix a missing statement for
HUGETLB), the TLB Refill handler was fixed so that non-OCTEON targets
would work properly with huge pages. The change was incorrect in that
it broke the OCTEON case.
The problem is shown here:
xxx0: df7a0000 ld k0,0(k1)
.
.
.
xxxc0: df610000 ld at,0(k1)
xxxc4: 335a0ff0 andi k0,k0,0xff0
xxxc8: e825ffcd bbit1 at,0x5,0x0
xxxcc: 003ad82d daddu k1,at,k0
.
.
.
In the non-octeon case there is a destructive test for the huge PTE
bit, and then at 0, $k0 is reloaded (that is what the 8393c524a25609
patch added).
In the octeon case, we modify k1 in the branch delay slot, but we
never need k0 again, so the new load is not needed, but since k1 is
modified, if we do the load, we load from a garbage location and then
get a nested TLB Refill, which is seen in userspace as either SIGBUS
or SIGSEGV (depending on the garbage).
The real fix is to only do this reloading if it is needed, and never
where it is harmful.
Signed-off-by: David Daney <david.daney@cavium.com>
Cc: Huacai Chen <chenhc@lemote.com>
Cc: Fuxin Zhang <zhangfx@lemote.com>
Cc: Zhangjin Wu <wuzhangjin@gmail.com>
Cc: stable@vger.kernel.org
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/8151/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2014-10-20 16:34:23 -06:00
|
|
|
htlb_info.need_reload_pte = true;
|
2010-12-27 19:07:57 -07:00
|
|
|
vmalloc_mode = refill_noscratch;
|
|
|
|
/*
|
|
|
|
* create the plain linear handler
|
|
|
|
*/
|
|
|
|
if (bcm1250_m3_war()) {
|
|
|
|
unsigned int segbits = 44;
|
|
|
|
|
|
|
|
uasm_i_dmfc0(&p, K0, C0_BADVADDR);
|
|
|
|
uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
|
|
|
|
uasm_i_xor(&p, K0, K0, K1);
|
|
|
|
uasm_i_dsrl_safe(&p, K1, K0, 62);
|
|
|
|
uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
|
|
|
|
uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
|
|
|
|
uasm_i_or(&p, K0, K0, K1);
|
|
|
|
uasm_il_bnez(&p, &r, K0, label_leave);
|
|
|
|
/* No need for uasm_i_nop */
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2005-09-03 16:56:16 -06:00
|
|
|
#ifdef CONFIG_64BIT
|
2010-12-27 19:07:57 -07:00
|
|
|
build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
|
2005-04-16 16:20:36 -06:00
|
|
|
#else
|
2010-12-27 19:07:57 -07:00
|
|
|
build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
|
2012-10-16 16:48:10 -06:00
|
|
|
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
|
2010-12-27 19:07:57 -07:00
|
|
|
build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update);
|
2009-05-27 18:47:44 -06:00
|
|
|
#endif
|
|
|
|
|
2010-12-27 19:07:57 -07:00
|
|
|
build_get_ptep(&p, K0, K1);
|
|
|
|
build_update_entries(&p, K0, K1);
|
|
|
|
build_tlb_write_entry(&p, &l, &r, tlb_random);
|
|
|
|
uasm_l_leave(&l, p);
|
|
|
|
uasm_i_eret(&p); /* return from trap */
|
|
|
|
}
|
2012-10-16 16:48:10 -06:00
|
|
|
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
|
2009-05-27 18:47:44 -06:00
|
|
|
uasm_l_tlb_huge_update(&l, p);
|
MIPS: tlbex: Properly fix HUGE TLB Refill exception handler
In commit 8393c524a25609 (MIPS: tlbex: Fix a missing statement for
HUGETLB), the TLB Refill handler was fixed so that non-OCTEON targets
would work properly with huge pages. The change was incorrect in that
it broke the OCTEON case.
The problem is shown here:
xxx0: df7a0000 ld k0,0(k1)
.
.
.
xxxc0: df610000 ld at,0(k1)
xxxc4: 335a0ff0 andi k0,k0,0xff0
xxxc8: e825ffcd bbit1 at,0x5,0x0
xxxcc: 003ad82d daddu k1,at,k0
.
.
.
In the non-octeon case there is a destructive test for the huge PTE
bit, and then at 0, $k0 is reloaded (that is what the 8393c524a25609
patch added).
In the octeon case, we modify k1 in the branch delay slot, but we
never need k0 again, so the new load is not needed, but since k1 is
modified, if we do the load, we load from a garbage location and then
get a nested TLB Refill, which is seen in userspace as either SIGBUS
or SIGSEGV (depending on the garbage).
The real fix is to only do this reloading if it is needed, and never
where it is harmful.
Signed-off-by: David Daney <david.daney@cavium.com>
Cc: Huacai Chen <chenhc@lemote.com>
Cc: Fuxin Zhang <zhangfx@lemote.com>
Cc: Zhangjin Wu <wuzhangjin@gmail.com>
Cc: stable@vger.kernel.org
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/8151/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2014-10-20 16:34:23 -06:00
|
|
|
if (htlb_info.need_reload_pte)
|
|
|
|
UASM_i_LW(&p, htlb_info.huge_pte, 0, K1);
|
2010-12-27 19:07:57 -07:00
|
|
|
build_huge_update_entries(&p, htlb_info.huge_pte, K1);
|
|
|
|
build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random,
|
|
|
|
htlb_info.restore_scratch);
|
2009-05-27 18:47:44 -06:00
|
|
|
#endif
|
|
|
|
|
2005-09-03 16:56:16 -06:00
|
|
|
#ifdef CONFIG_64BIT
|
2010-12-27 19:07:57 -07:00
|
|
|
build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode);
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Overflow check: For the 64bit handler, we need at least one
|
|
|
|
* free instruction slot for the wrap-around branch. In worst
|
|
|
|
* case, if the intended insertion point is a delay slot, we
|
2006-10-03 14:21:02 -06:00
|
|
|
* need three, with the second nop'ed and the third being
|
2005-04-16 16:20:36 -06:00
|
|
|
* unused.
|
|
|
|
*/
|
2013-09-25 10:21:26 -06:00
|
|
|
switch (boot_cpu_type()) {
|
|
|
|
default:
|
|
|
|
if (sizeof(long) == 4) {
|
|
|
|
case CPU_LOONGSON2:
|
|
|
|
/* Loongson2 ebase is different than r4k, we have more space */
|
|
|
|
if ((p - tlb_handler) > 64)
|
|
|
|
panic("TLB refill handler space exceeded");
|
2009-05-20 12:40:59 -06:00
|
|
|
/*
|
2013-09-25 10:21:26 -06:00
|
|
|
* Now fold the handler in the TLB refill handler space.
|
2009-05-20 12:40:59 -06:00
|
|
|
*/
|
2013-09-25 10:21:26 -06:00
|
|
|
f = final_handler;
|
|
|
|
/* Simplest case, just copy the handler. */
|
|
|
|
uasm_copy_handler(relocs, labels, tlb_handler, p, f);
|
|
|
|
final_len = p - tlb_handler;
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1)
|
|
|
|
|| (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3)
|
|
|
|
&& uasm_insn_has_bdelay(relocs,
|
|
|
|
tlb_handler + MIPS64_REFILL_INSNS - 3)))
|
|
|
|
panic("TLB refill handler space exceeded");
|
2009-05-20 12:40:59 -06:00
|
|
|
/*
|
2013-09-25 10:21:26 -06:00
|
|
|
* Now fold the handler in the TLB refill handler space.
|
2009-05-20 12:40:59 -06:00
|
|
|
*/
|
2013-09-25 10:21:26 -06:00
|
|
|
f = final_handler + MIPS64_REFILL_INSNS;
|
|
|
|
if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) {
|
|
|
|
/* Just copy the handler. */
|
|
|
|
uasm_copy_handler(relocs, labels, tlb_handler, p, f);
|
|
|
|
final_len = p - tlb_handler;
|
|
|
|
} else {
|
|
|
|
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
|
|
|
|
const enum label_id ls = label_tlb_huge_update;
|
|
|
|
#else
|
|
|
|
const enum label_id ls = label_vmalloc;
|
|
|
|
#endif
|
|
|
|
u32 *split;
|
|
|
|
int ov = 0;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++)
|
|
|
|
;
|
|
|
|
BUG_ON(i == ARRAY_SIZE(labels));
|
|
|
|
split = labels[i].addr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* See if we have overflown one way or the other.
|
|
|
|
*/
|
|
|
|
if (split > tlb_handler + MIPS64_REFILL_INSNS ||
|
|
|
|
split < p - MIPS64_REFILL_INSNS)
|
|
|
|
ov = 1;
|
|
|
|
|
|
|
|
if (ov) {
|
|
|
|
/*
|
|
|
|
* Split two instructions before the end. One
|
|
|
|
* for the branch and one for the instruction
|
|
|
|
* in the delay slot.
|
|
|
|
*/
|
|
|
|
split = tlb_handler + MIPS64_REFILL_INSNS - 2;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the branch would fall in a delay slot,
|
|
|
|
* we must back up an additional instruction
|
|
|
|
* so that it is no longer in a delay slot.
|
|
|
|
*/
|
|
|
|
if (uasm_insn_has_bdelay(relocs, split - 1))
|
|
|
|
split--;
|
|
|
|
}
|
|
|
|
/* Copy first part of the handler. */
|
|
|
|
uasm_copy_handler(relocs, labels, tlb_handler, split, f);
|
|
|
|
f += split - tlb_handler;
|
|
|
|
|
|
|
|
if (ov) {
|
|
|
|
/* Insert branch. */
|
|
|
|
uasm_l_split(&l, final_handler);
|
|
|
|
uasm_il_b(&f, &r, label_split);
|
|
|
|
if (uasm_insn_has_bdelay(relocs, split))
|
|
|
|
uasm_i_nop(&f);
|
|
|
|
else {
|
|
|
|
uasm_copy_handler(relocs, labels,
|
|
|
|
split, split + 1, f);
|
|
|
|
uasm_move_labels(labels, f, f + 1, -1);
|
|
|
|
f++;
|
|
|
|
split++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Copy the rest of the handler. */
|
|
|
|
uasm_copy_handler(relocs, labels, split, p, final_handler);
|
|
|
|
final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) +
|
|
|
|
(p - split);
|
2009-05-20 12:40:59 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2013-09-25 10:21:26 -06:00
|
|
|
break;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_resolve_relocs(relocs, labels);
|
|
|
|
pr_debug("Wrote TLB refill handler (%u instructions).\n",
|
|
|
|
final_len);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-03-29 10:53:00 -07:00
|
|
|
memcpy((void *)ebase, final_handler, 0x100);
|
2014-07-11 16:18:05 -06:00
|
|
|
local_flush_icache_range(ebase, ebase + 0x100);
|
2007-10-18 01:11:17 -06:00
|
|
|
|
2012-10-16 14:20:26 -06:00
|
|
|
dump_handler("r4000_tlb_refill", (u32 *)ebase, 64);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2016-03-02 18:45:12 -07:00
|
|
|
static void setup_pw(void)
|
|
|
|
{
|
|
|
|
unsigned long pgd_i, pgd_w;
|
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
|
|
|
unsigned long pmd_i, pmd_w;
|
|
|
|
#endif
|
|
|
|
unsigned long pt_i, pt_w;
|
|
|
|
unsigned long pte_i, pte_w;
|
|
|
|
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
|
|
|
|
unsigned long psn;
|
|
|
|
|
|
|
|
psn = ilog2(_PAGE_HUGE); /* bit used to indicate huge page */
|
|
|
|
#endif
|
|
|
|
pgd_i = PGDIR_SHIFT; /* 1st level PGD */
|
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
|
|
|
pgd_w = PGDIR_SHIFT - PMD_SHIFT + PGD_ORDER;
|
|
|
|
|
|
|
|
pmd_i = PMD_SHIFT; /* 2nd level PMD */
|
|
|
|
pmd_w = PMD_SHIFT - PAGE_SHIFT;
|
|
|
|
#else
|
|
|
|
pgd_w = PGDIR_SHIFT - PAGE_SHIFT + PGD_ORDER;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
pt_i = PAGE_SHIFT; /* 3rd level PTE */
|
|
|
|
pt_w = PAGE_SHIFT - 3;
|
|
|
|
|
|
|
|
pte_i = ilog2(_PAGE_GLOBAL);
|
|
|
|
pte_w = 0;
|
|
|
|
|
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
|
|
|
write_c0_pwfield(pgd_i << 24 | pmd_i << 12 | pt_i << 6 | pte_i);
|
|
|
|
write_c0_pwsize(1 << 30 | pgd_w << 24 | pmd_w << 12 | pt_w << 6 | pte_w);
|
|
|
|
#else
|
|
|
|
write_c0_pwfield(pgd_i << 24 | pt_i << 6 | pte_i);
|
|
|
|
write_c0_pwsize(1 << 30 | pgd_w << 24 | pt_w << 6 | pte_w);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
|
|
|
|
write_c0_pwctl(1 << 6 | psn);
|
|
|
|
#endif
|
|
|
|
write_c0_kpgd(swapper_pg_dir);
|
|
|
|
kscratch_used_mask |= (1 << 7); /* KScratch6 is used for KPGD */
|
|
|
|
}
|
|
|
|
|
|
|
|
static void build_loongson3_tlb_refill_handler(void)
|
|
|
|
{
|
|
|
|
u32 *p = tlb_handler;
|
|
|
|
struct uasm_label *l = labels;
|
|
|
|
struct uasm_reloc *r = relocs;
|
|
|
|
|
|
|
|
memset(labels, 0, sizeof(labels));
|
|
|
|
memset(relocs, 0, sizeof(relocs));
|
|
|
|
memset(tlb_handler, 0, sizeof(tlb_handler));
|
|
|
|
|
|
|
|
if (check_for_high_segbits) {
|
|
|
|
uasm_i_dmfc0(&p, K0, C0_BADVADDR);
|
|
|
|
uasm_i_dsrl_safe(&p, K1, K0, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
|
|
|
|
uasm_il_beqz(&p, &r, K1, label_vmalloc);
|
|
|
|
uasm_i_nop(&p);
|
|
|
|
|
|
|
|
uasm_il_bgez(&p, &r, K0, label_large_segbits_fault);
|
|
|
|
uasm_i_nop(&p);
|
|
|
|
uasm_l_vmalloc(&l, p);
|
|
|
|
}
|
|
|
|
|
|
|
|
uasm_i_dmfc0(&p, K1, C0_PGD);
|
|
|
|
|
|
|
|
uasm_i_lddir(&p, K0, K1, 3); /* global page dir */
|
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
|
|
|
uasm_i_lddir(&p, K1, K0, 1); /* middle page dir */
|
|
|
|
#endif
|
|
|
|
uasm_i_ldpte(&p, K1, 0); /* even */
|
|
|
|
uasm_i_ldpte(&p, K1, 1); /* odd */
|
|
|
|
uasm_i_tlbwr(&p);
|
|
|
|
|
|
|
|
/* restore page mask */
|
|
|
|
if (PM_DEFAULT_MASK >> 16) {
|
|
|
|
uasm_i_lui(&p, K0, PM_DEFAULT_MASK >> 16);
|
|
|
|
uasm_i_ori(&p, K0, K0, PM_DEFAULT_MASK & 0xffff);
|
|
|
|
uasm_i_mtc0(&p, K0, C0_PAGEMASK);
|
|
|
|
} else if (PM_DEFAULT_MASK) {
|
|
|
|
uasm_i_ori(&p, K0, 0, PM_DEFAULT_MASK);
|
|
|
|
uasm_i_mtc0(&p, K0, C0_PAGEMASK);
|
|
|
|
} else {
|
|
|
|
uasm_i_mtc0(&p, 0, C0_PAGEMASK);
|
|
|
|
}
|
|
|
|
|
|
|
|
uasm_i_eret(&p);
|
|
|
|
|
|
|
|
if (check_for_high_segbits) {
|
|
|
|
uasm_l_large_segbits_fault(&l, p);
|
|
|
|
UASM_i_LA(&p, K1, (unsigned long)tlb_do_page_fault_0);
|
|
|
|
uasm_i_jr(&p, K1);
|
|
|
|
uasm_i_nop(&p);
|
|
|
|
}
|
|
|
|
|
|
|
|
uasm_resolve_relocs(relocs, labels);
|
|
|
|
memcpy((void *)(ebase + 0x80), tlb_handler, 0x80);
|
|
|
|
local_flush_icache_range(ebase + 0x80, ebase + 0x100);
|
|
|
|
dump_handler("loongson3_tlb_refill", (u32 *)(ebase + 0x80), 32);
|
|
|
|
}
|
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
extern u32 handle_tlbl[], handle_tlbl_end[];
|
|
|
|
extern u32 handle_tlbs[], handle_tlbs_end[];
|
|
|
|
extern u32 handle_tlbm[], handle_tlbm_end[];
|
2014-04-10 13:06:17 -06:00
|
|
|
extern u32 tlbmiss_handler_setup_pgd_start[], tlbmiss_handler_setup_pgd[];
|
|
|
|
extern u32 tlbmiss_handler_setup_pgd_end[];
|
2010-12-21 15:19:11 -07:00
|
|
|
|
2013-09-25 04:58:04 -06:00
|
|
|
static void build_setup_pgd(void)
|
2010-12-21 15:19:11 -07:00
|
|
|
{
|
|
|
|
const int a0 = 4;
|
2013-09-25 04:58:04 -06:00
|
|
|
const int __maybe_unused a1 = 5;
|
|
|
|
const int __maybe_unused a2 = 6;
|
2014-04-10 13:06:17 -06:00
|
|
|
u32 *p = tlbmiss_handler_setup_pgd_start;
|
2013-06-23 11:16:19 -06:00
|
|
|
const int tlbmiss_handler_setup_pgd_size =
|
2014-04-10 13:06:17 -06:00
|
|
|
tlbmiss_handler_setup_pgd_end - tlbmiss_handler_setup_pgd_start;
|
2013-09-25 04:58:04 -06:00
|
|
|
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
|
|
|
|
long pgdc = (long)pgd_current;
|
|
|
|
#endif
|
2010-12-21 15:19:11 -07:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
memset(tlbmiss_handler_setup_pgd, 0, tlbmiss_handler_setup_pgd_size *
|
|
|
|
sizeof(tlbmiss_handler_setup_pgd[0]));
|
2010-12-21 15:19:11 -07:00
|
|
|
memset(labels, 0, sizeof(labels));
|
|
|
|
memset(relocs, 0, sizeof(relocs));
|
|
|
|
pgd_reg = allocate_kscratch();
|
2013-09-25 04:58:04 -06:00
|
|
|
#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
|
2010-12-21 15:19:11 -07:00
|
|
|
if (pgd_reg == -1) {
|
2013-09-25 04:58:04 -06:00
|
|
|
struct uasm_label *l = labels;
|
|
|
|
struct uasm_reloc *r = relocs;
|
|
|
|
|
2010-12-21 15:19:11 -07:00
|
|
|
/* PGD << 11 in c0_Context */
|
|
|
|
/*
|
|
|
|
* If it is a ckseg0 address, convert to a physical
|
|
|
|
* address. Shifting right by 29 and adding 4 will
|
|
|
|
* result in zero for these addresses.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
UASM_i_SRA(&p, a1, a0, 29);
|
|
|
|
UASM_i_ADDIU(&p, a1, a1, 4);
|
|
|
|
uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1);
|
|
|
|
uasm_i_nop(&p);
|
|
|
|
uasm_i_dinsm(&p, a0, 0, 29, 64 - 29);
|
|
|
|
uasm_l_tlbl_goaround1(&l, p);
|
|
|
|
UASM_i_SLL(&p, a0, a0, 11);
|
|
|
|
uasm_i_jr(&p, 31);
|
|
|
|
UASM_i_MTC0(&p, a0, C0_CONTEXT);
|
|
|
|
} else {
|
|
|
|
/* PGD in c0_KScratch */
|
|
|
|
uasm_i_jr(&p, 31);
|
2016-03-02 18:45:12 -07:00
|
|
|
if (cpu_has_ldpte)
|
|
|
|
UASM_i_MTC0(&p, a0, C0_PWBASE);
|
|
|
|
else
|
|
|
|
UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
|
2010-12-21 15:19:11 -07:00
|
|
|
}
|
2013-09-25 04:58:04 -06:00
|
|
|
#else
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
/* Save PGD to pgd_current[smp_processor_id()] */
|
|
|
|
UASM_i_CPUID_MFC0(&p, a1, SMP_CPUID_REG);
|
|
|
|
UASM_i_SRL_SAFE(&p, a1, a1, SMP_CPUID_PTRSHIFT);
|
|
|
|
UASM_i_LA_mostly(&p, a2, pgdc);
|
|
|
|
UASM_i_ADDU(&p, a2, a2, a1);
|
|
|
|
UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2);
|
|
|
|
#else
|
|
|
|
UASM_i_LA_mostly(&p, a2, pgdc);
|
|
|
|
UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2);
|
|
|
|
#endif /* SMP */
|
|
|
|
uasm_i_jr(&p, 31);
|
|
|
|
|
|
|
|
/* if pgd_reg is allocated, save PGD also to scratch register */
|
|
|
|
if (pgd_reg != -1)
|
|
|
|
UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
|
|
|
|
else
|
|
|
|
uasm_i_nop(&p);
|
|
|
|
#endif
|
2013-06-23 11:16:19 -06:00
|
|
|
if (p >= tlbmiss_handler_setup_pgd_end)
|
|
|
|
panic("tlbmiss_handler_setup_pgd space exceeded");
|
|
|
|
|
2010-12-21 15:19:11 -07:00
|
|
|
uasm_resolve_relocs(relocs, labels);
|
2013-06-23 11:16:19 -06:00
|
|
|
pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n",
|
|
|
|
(unsigned int)(p - tlbmiss_handler_setup_pgd));
|
2010-12-21 15:19:11 -07:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd,
|
|
|
|
tlbmiss_handler_setup_pgd_size);
|
2010-12-21 15:19:11 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void
|
2009-05-08 16:10:50 -06:00
|
|
|
iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
#ifdef CONFIG_SMP
|
2014-11-21 16:16:48 -07:00
|
|
|
# ifdef CONFIG_PHYS_ADDR_T_64BIT
|
2005-04-16 16:20:36 -06:00
|
|
|
if (cpu_has_64bits)
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_lld(p, pte, 0, ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
else
|
|
|
|
# endif
|
2008-01-28 13:05:38 -07:00
|
|
|
UASM_i_LL(p, pte, 0, ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
#else
|
2014-11-21 16:16:48 -07:00
|
|
|
# ifdef CONFIG_PHYS_ADDR_T_64BIT
|
2005-04-16 16:20:36 -06:00
|
|
|
if (cpu_has_64bits)
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_ld(p, pte, 0, ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
else
|
|
|
|
# endif
|
2008-01-28 13:05:38 -07:00
|
|
|
UASM_i_LW(p, pte, 0, ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void
|
2008-01-28 13:05:38 -07:00
|
|
|
iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
|
2016-04-19 02:25:07 -06:00
|
|
|
unsigned int mode, unsigned int scratch)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2005-04-28 02:52:57 -06:00
|
|
|
unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
|
2016-04-19 02:25:08 -06:00
|
|
|
unsigned int swmode = mode & ~hwmode;
|
2005-04-28 02:52:57 -06:00
|
|
|
|
2016-04-19 02:25:05 -06:00
|
|
|
if (config_enabled(CONFIG_XPA) && !cpu_has_64bits) {
|
2016-04-19 02:25:08 -06:00
|
|
|
uasm_i_lui(p, scratch, swmode >> 16);
|
2015-02-26 17:16:38 -07:00
|
|
|
uasm_i_or(p, pte, pte, scratch);
|
2016-04-19 02:25:08 -06:00
|
|
|
BUG_ON(swmode & 0xffff);
|
|
|
|
} else {
|
|
|
|
uasm_i_ori(p, pte, pte, mode);
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
#ifdef CONFIG_SMP
|
2014-11-21 16:16:48 -07:00
|
|
|
# ifdef CONFIG_PHYS_ADDR_T_64BIT
|
2005-04-16 16:20:36 -06:00
|
|
|
if (cpu_has_64bits)
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_scd(p, pte, 0, ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
else
|
|
|
|
# endif
|
2008-01-28 13:05:38 -07:00
|
|
|
UASM_i_SC(p, pte, 0, ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (r10000_llsc_war())
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_il_beqzl(p, r, pte, label_smp_pgtable_change);
|
2005-04-16 16:20:36 -06:00
|
|
|
else
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2014-11-21 16:16:48 -07:00
|
|
|
# ifdef CONFIG_PHYS_ADDR_T_64BIT
|
2005-04-16 16:20:36 -06:00
|
|
|
if (!cpu_has_64bits) {
|
2008-01-28 13:05:38 -07:00
|
|
|
/* no uasm_i_nop needed */
|
|
|
|
uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr);
|
|
|
|
uasm_i_ori(p, pte, pte, hwmode);
|
2016-04-19 02:25:08 -06:00
|
|
|
BUG_ON(hwmode & ~0xffff);
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr);
|
|
|
|
uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
|
|
|
|
/* no uasm_i_nop needed */
|
|
|
|
uasm_i_lw(p, pte, 0, ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
} else
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(p);
|
2005-04-16 16:20:36 -06:00
|
|
|
# else
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(p);
|
2005-04-16 16:20:36 -06:00
|
|
|
# endif
|
|
|
|
#else
|
2014-11-21 16:16:48 -07:00
|
|
|
# ifdef CONFIG_PHYS_ADDR_T_64BIT
|
2005-04-16 16:20:36 -06:00
|
|
|
if (cpu_has_64bits)
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_sd(p, pte, 0, ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
else
|
|
|
|
# endif
|
2008-01-28 13:05:38 -07:00
|
|
|
UASM_i_SW(p, pte, 0, ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2014-11-21 16:16:48 -07:00
|
|
|
# ifdef CONFIG_PHYS_ADDR_T_64BIT
|
2005-04-16 16:20:36 -06:00
|
|
|
if (!cpu_has_64bits) {
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr);
|
|
|
|
uasm_i_ori(p, pte, pte, hwmode);
|
2016-04-19 02:25:08 -06:00
|
|
|
BUG_ON(hwmode & ~0xffff);
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr);
|
|
|
|
uasm_i_lw(p, pte, 0, ptr);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
# endif
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check if PTE is present, if not then jump to LABEL. PTR points to
|
|
|
|
* the page table where this PTE is located, PTE will be re-loaded
|
|
|
|
* with it's original value.
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void
|
2009-05-08 16:10:50 -06:00
|
|
|
build_pte_present(u32 **p, struct uasm_reloc **r,
|
2011-07-05 17:34:46 -06:00
|
|
|
int pte, int ptr, int scratch, enum label_id lid)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2011-07-05 17:34:46 -06:00
|
|
|
int t = scratch >= 0 ? scratch : pte;
|
2015-04-27 08:07:18 -06:00
|
|
|
int cur = pte;
|
2011-07-05 17:34:46 -06:00
|
|
|
|
2012-09-13 15:51:46 -06:00
|
|
|
if (cpu_has_rixi) {
|
2010-12-20 16:54:50 -07:00
|
|
|
if (use_bbit_insns()) {
|
|
|
|
uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
|
|
|
|
uasm_i_nop(p);
|
|
|
|
} else {
|
2015-04-27 08:07:18 -06:00
|
|
|
if (_PAGE_PRESENT_SHIFT) {
|
|
|
|
uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT);
|
|
|
|
cur = t;
|
|
|
|
}
|
|
|
|
uasm_i_andi(p, t, cur, 1);
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_il_beqz(p, r, t, lid);
|
|
|
|
if (pte == t)
|
|
|
|
/* You lose the SMP race :-(*/
|
|
|
|
iPTE_LW(p, pte, ptr);
|
2010-12-20 16:54:50 -07:00
|
|
|
}
|
2010-02-10 16:12:47 -07:00
|
|
|
} else {
|
2015-04-27 08:07:18 -06:00
|
|
|
if (_PAGE_PRESENT_SHIFT) {
|
|
|
|
uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT);
|
|
|
|
cur = t;
|
|
|
|
}
|
|
|
|
uasm_i_andi(p, t, cur,
|
2016-04-19 02:25:03 -06:00
|
|
|
(_PAGE_PRESENT | _PAGE_NO_READ) >> _PAGE_PRESENT_SHIFT);
|
|
|
|
uasm_i_xori(p, t, t, _PAGE_PRESENT >> _PAGE_PRESENT_SHIFT);
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_il_bnez(p, r, t, lid);
|
|
|
|
if (pte == t)
|
|
|
|
/* You lose the SMP race :-(*/
|
|
|
|
iPTE_LW(p, pte, ptr);
|
2010-02-10 16:12:47 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Make PTE valid, store result in PTR. */
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void
|
2008-01-28 13:05:38 -07:00
|
|
|
build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
|
2016-04-19 02:25:07 -06:00
|
|
|
unsigned int ptr, unsigned int scratch)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2005-04-28 02:52:57 -06:00
|
|
|
unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;
|
|
|
|
|
2016-04-19 02:25:07 -06:00
|
|
|
iPTE_SW(p, r, pte, ptr, mode, scratch);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check if PTE can be written to, if not branch to LABEL. Regardless
|
|
|
|
* restore PTE with value from PTR when done.
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void
|
2009-05-08 16:10:50 -06:00
|
|
|
build_pte_writable(u32 **p, struct uasm_reloc **r,
|
2011-07-05 17:34:46 -06:00
|
|
|
unsigned int pte, unsigned int ptr, int scratch,
|
|
|
|
enum label_id lid)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2011-07-05 17:34:46 -06:00
|
|
|
int t = scratch >= 0 ? scratch : pte;
|
2015-04-27 08:07:18 -06:00
|
|
|
int cur = pte;
|
2011-07-05 17:34:46 -06:00
|
|
|
|
2015-04-27 08:07:18 -06:00
|
|
|
if (_PAGE_PRESENT_SHIFT) {
|
|
|
|
uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT);
|
|
|
|
cur = t;
|
|
|
|
}
|
|
|
|
uasm_i_andi(p, t, cur,
|
2015-04-27 08:07:17 -06:00
|
|
|
(_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT);
|
|
|
|
uasm_i_xori(p, t, t,
|
|
|
|
(_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT);
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_il_bnez(p, r, t, lid);
|
|
|
|
if (pte == t)
|
|
|
|
/* You lose the SMP race :-(*/
|
2010-12-20 16:54:50 -07:00
|
|
|
iPTE_LW(p, pte, ptr);
|
2011-07-05 17:34:46 -06:00
|
|
|
else
|
|
|
|
uasm_i_nop(p);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Make PTE writable, update software status bits as well, then store
|
|
|
|
* at PTR.
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void
|
2008-01-28 13:05:38 -07:00
|
|
|
build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
|
2016-04-19 02:25:07 -06:00
|
|
|
unsigned int ptr, unsigned int scratch)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2005-04-28 02:52:57 -06:00
|
|
|
unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
|
|
|
|
| _PAGE_DIRTY);
|
|
|
|
|
2016-04-19 02:25:07 -06:00
|
|
|
iPTE_SW(p, r, pte, ptr, mode, scratch);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check if PTE can be modified, if not branch to LABEL. Regardless
|
|
|
|
* restore PTE with value from PTR when done.
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void
|
2009-05-08 16:10:50 -06:00
|
|
|
build_pte_modifiable(u32 **p, struct uasm_reloc **r,
|
2011-07-05 17:34:46 -06:00
|
|
|
unsigned int pte, unsigned int ptr, int scratch,
|
|
|
|
enum label_id lid)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2010-12-20 16:54:50 -07:00
|
|
|
if (use_bbit_insns()) {
|
|
|
|
uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
|
|
|
|
uasm_i_nop(p);
|
|
|
|
} else {
|
2011-07-05 17:34:46 -06:00
|
|
|
int t = scratch >= 0 ? scratch : pte;
|
2015-02-26 17:16:38 -07:00
|
|
|
uasm_i_srl(p, t, pte, _PAGE_WRITE_SHIFT);
|
|
|
|
uasm_i_andi(p, t, t, 1);
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_il_beqz(p, r, t, lid);
|
|
|
|
if (pte == t)
|
|
|
|
/* You lose the SMP race :-(*/
|
|
|
|
iPTE_LW(p, pte, ptr);
|
2010-12-20 16:54:50 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2009-10-14 13:16:56 -06:00
|
|
|
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
|
2010-12-21 15:19:11 -07:00
|
|
|
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* R3000 style TLB load/store/modify handlers.
|
|
|
|
*/
|
|
|
|
|
2005-06-13 14:24:00 -06:00
|
|
|
/*
|
|
|
|
* This places the pte into ENTRYLO0 and writes it with tlbwi.
|
|
|
|
* Then it returns.
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void
|
2005-06-13 14:24:00 -06:00
|
|
|
build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
|
|
|
|
uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
|
|
|
|
uasm_i_tlbwi(p);
|
|
|
|
uasm_i_jr(p, tmp);
|
|
|
|
uasm_i_rfe(p); /* branch delay */
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2005-06-13 14:24:00 -06:00
|
|
|
* This places the pte into ENTRYLO0 and writes it with tlbwi
|
|
|
|
* or tlbwr as appropriate. This is because the index register
|
|
|
|
* may have the probe fail bit set as a result of a trap on a
|
|
|
|
* kseg2 access, i.e. without refill. Then it returns.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void
|
2008-01-28 13:05:38 -07:00
|
|
|
build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
|
|
|
|
struct uasm_reloc **r, unsigned int pte,
|
|
|
|
unsigned int tmp)
|
|
|
|
{
|
|
|
|
uasm_i_mfc0(p, tmp, C0_INDEX);
|
|
|
|
uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
|
|
|
|
uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
|
|
|
|
uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */
|
|
|
|
uasm_i_tlbwi(p); /* cp0 delay */
|
|
|
|
uasm_i_jr(p, tmp);
|
|
|
|
uasm_i_rfe(p); /* branch delay */
|
|
|
|
uasm_l_r3000_write_probe_fail(l, *p);
|
|
|
|
uasm_i_tlbwr(p); /* cp0 delay */
|
|
|
|
uasm_i_jr(p, tmp);
|
|
|
|
uasm_i_rfe(p); /* branch delay */
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void
|
2005-04-16 16:20:36 -06:00
|
|
|
build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
|
|
|
|
unsigned int ptr)
|
|
|
|
{
|
|
|
|
long pgdc = (long)pgd_current;
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_mfc0(p, pte, C0_BADVADDR);
|
|
|
|
uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */
|
|
|
|
uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
|
|
|
|
uasm_i_srl(p, pte, pte, 22); /* load delay */
|
|
|
|
uasm_i_sll(p, pte, pte, 2);
|
|
|
|
uasm_i_addu(p, ptr, ptr, pte);
|
|
|
|
uasm_i_mfc0(p, pte, C0_CONTEXT);
|
|
|
|
uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */
|
|
|
|
uasm_i_andi(p, pte, pte, 0xffc); /* load delay */
|
|
|
|
uasm_i_addu(p, ptr, ptr, pte);
|
|
|
|
uasm_i_lw(p, pte, 0, ptr);
|
|
|
|
uasm_i_tlbp(p); /* load delay */
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_r3000_tlb_load_handler(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
u32 *p = handle_tlbl;
|
2013-06-23 11:16:19 -06:00
|
|
|
const int handle_tlbl_size = handle_tlbl_end - handle_tlbl;
|
2008-01-28 13:05:38 -07:00
|
|
|
struct uasm_label *l = labels;
|
|
|
|
struct uasm_reloc *r = relocs;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0]));
|
2005-04-16 16:20:36 -06:00
|
|
|
memset(labels, 0, sizeof(labels));
|
|
|
|
memset(relocs, 0, sizeof(relocs));
|
|
|
|
|
|
|
|
build_r3000_tlbchange_handler_head(&p, K0, K1);
|
2011-07-05 17:34:46 -06:00
|
|
|
build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl);
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(&p); /* load delay */
|
2016-04-19 02:25:07 -06:00
|
|
|
build_make_valid(&p, &r, K0, K1, -1);
|
2005-06-13 14:24:00 -06:00
|
|
|
build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_l_nopage_tlbl(&l, p);
|
|
|
|
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
|
|
|
|
uasm_i_nop(&p);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
if (p >= handle_tlbl_end)
|
2005-04-16 16:20:36 -06:00
|
|
|
panic("TLB load handler fastpath space exceeded");
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_resolve_relocs(relocs, labels);
|
|
|
|
pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
|
|
|
|
(unsigned int)(p - handle_tlbl));
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
dump_handler("r3000_tlb_load", handle_tlbl, handle_tlbl_size);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_r3000_tlb_store_handler(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
u32 *p = handle_tlbs;
|
2013-06-23 11:16:19 -06:00
|
|
|
const int handle_tlbs_size = handle_tlbs_end - handle_tlbs;
|
2008-01-28 13:05:38 -07:00
|
|
|
struct uasm_label *l = labels;
|
|
|
|
struct uasm_reloc *r = relocs;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0]));
|
2005-04-16 16:20:36 -06:00
|
|
|
memset(labels, 0, sizeof(labels));
|
|
|
|
memset(relocs, 0, sizeof(relocs));
|
|
|
|
|
|
|
|
build_r3000_tlbchange_handler_head(&p, K0, K1);
|
2011-07-05 17:34:46 -06:00
|
|
|
build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs);
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(&p); /* load delay */
|
2016-04-19 02:25:07 -06:00
|
|
|
build_make_write(&p, &r, K0, K1, -1);
|
2005-06-13 14:24:00 -06:00
|
|
|
build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_l_nopage_tlbs(&l, p);
|
|
|
|
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
|
|
|
|
uasm_i_nop(&p);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-07-18 03:45:47 -06:00
|
|
|
if (p >= handle_tlbs_end)
|
2005-04-16 16:20:36 -06:00
|
|
|
panic("TLB store handler fastpath space exceeded");
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_resolve_relocs(relocs, labels);
|
|
|
|
pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
|
|
|
|
(unsigned int)(p - handle_tlbs));
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
dump_handler("r3000_tlb_store", handle_tlbs, handle_tlbs_size);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_r3000_tlb_modify_handler(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
u32 *p = handle_tlbm;
|
2013-06-23 11:16:19 -06:00
|
|
|
const int handle_tlbm_size = handle_tlbm_end - handle_tlbm;
|
2008-01-28 13:05:38 -07:00
|
|
|
struct uasm_label *l = labels;
|
|
|
|
struct uasm_reloc *r = relocs;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0]));
|
2005-04-16 16:20:36 -06:00
|
|
|
memset(labels, 0, sizeof(labels));
|
|
|
|
memset(relocs, 0, sizeof(relocs));
|
|
|
|
|
|
|
|
build_r3000_tlbchange_handler_head(&p, K0, K1);
|
2011-08-02 15:52:48 -06:00
|
|
|
build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm);
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_nop(&p); /* load delay */
|
2016-04-19 02:25:07 -06:00
|
|
|
build_make_write(&p, &r, K0, K1, -1);
|
2005-06-13 14:24:00 -06:00
|
|
|
build_r3000_pte_reload_tlbwi(&p, K0, K1);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_l_nopage_tlbm(&l, p);
|
|
|
|
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
|
|
|
|
uasm_i_nop(&p);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
if (p >= handle_tlbm_end)
|
2005-04-16 16:20:36 -06:00
|
|
|
panic("TLB modify handler fastpath space exceeded");
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_resolve_relocs(relocs, labels);
|
|
|
|
pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
|
|
|
|
(unsigned int)(p - handle_tlbm));
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
dump_handler("r3000_tlb_modify", handle_tlbm, handle_tlbm_size);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2009-10-14 13:16:56 -06:00
|
|
|
#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* R4000 style TLB load/store/modify handlers.
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static struct work_registers
|
2008-01-28 13:05:38 -07:00
|
|
|
build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
|
2011-07-05 17:34:46 -06:00
|
|
|
struct uasm_reloc **r)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2011-07-05 17:34:46 -06:00
|
|
|
struct work_registers wr = build_get_work_registers(p);
|
|
|
|
|
2005-09-03 16:56:16 -06:00
|
|
|
#ifdef CONFIG_64BIT
|
2011-07-05 17:34:46 -06:00
|
|
|
build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */
|
2005-04-16 16:20:36 -06:00
|
|
|
#else
|
2011-07-05 17:34:46 -06:00
|
|
|
build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
|
2012-10-16 16:48:10 -06:00
|
|
|
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
|
2009-05-27 18:47:44 -06:00
|
|
|
/*
|
|
|
|
* For huge tlb entries, pmd doesn't contain an address but
|
|
|
|
* instead contains the tlb pte. Check the PAGE_HUGE bit and
|
|
|
|
* see if we need to jump to huge tlb processing.
|
|
|
|
*/
|
2011-07-05 17:34:46 -06:00
|
|
|
build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update);
|
2009-05-27 18:47:44 -06:00
|
|
|
#endif
|
|
|
|
|
2011-07-05 17:34:46 -06:00
|
|
|
UASM_i_MFC0(p, wr.r1, C0_BADVADDR);
|
|
|
|
UASM_i_LW(p, wr.r2, 0, wr.r2);
|
|
|
|
UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
|
|
|
|
uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
|
|
|
|
UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_l_smp_pgtable_change(l, *p);
|
|
|
|
#endif
|
2011-07-05 17:34:46 -06:00
|
|
|
iPTE_LW(p, wr.r1, wr.r2); /* get even pte */
|
2014-11-27 04:13:08 -07:00
|
|
|
if (!m4kc_tlbp_war()) {
|
2006-08-23 07:26:50 -06:00
|
|
|
build_tlb_probe_entry(p);
|
2014-11-27 04:13:08 -07:00
|
|
|
if (cpu_has_htw) {
|
|
|
|
/* race condition happens, leaving */
|
|
|
|
uasm_i_ehb(p);
|
|
|
|
uasm_i_mfc0(p, wr.r3, C0_INDEX);
|
|
|
|
uasm_il_bltz(p, r, wr.r3, label_leave);
|
|
|
|
uasm_i_nop(p);
|
|
|
|
}
|
|
|
|
}
|
2011-07-05 17:34:46 -06:00
|
|
|
return wr;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void
|
2008-01-28 13:05:38 -07:00
|
|
|
build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
|
|
|
|
struct uasm_reloc **r, unsigned int tmp,
|
2005-04-16 16:20:36 -06:00
|
|
|
unsigned int ptr)
|
|
|
|
{
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_ori(p, ptr, ptr, sizeof(pte_t));
|
|
|
|
uasm_i_xori(p, ptr, ptr, sizeof(pte_t));
|
2005-04-16 16:20:36 -06:00
|
|
|
build_update_entries(p, tmp, ptr);
|
|
|
|
build_tlb_write_entry(p, l, r, tlb_indexed);
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_l_leave(l, *p);
|
2011-07-05 17:34:46 -06:00
|
|
|
build_restore_work_registers(p);
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_eret(p); /* return from trap */
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2005-09-03 16:56:16 -06:00
|
|
|
#ifdef CONFIG_64BIT
|
2010-04-28 13:16:18 -06:00
|
|
|
build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill);
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_r4000_tlb_load_handler(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
u32 *p = handle_tlbl;
|
2013-06-23 11:16:19 -06:00
|
|
|
const int handle_tlbl_size = handle_tlbl_end - handle_tlbl;
|
2008-01-28 13:05:38 -07:00
|
|
|
struct uasm_label *l = labels;
|
|
|
|
struct uasm_reloc *r = relocs;
|
2011-07-05 17:34:46 -06:00
|
|
|
struct work_registers wr;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0]));
|
2005-04-16 16:20:36 -06:00
|
|
|
memset(labels, 0, sizeof(labels));
|
|
|
|
memset(relocs, 0, sizeof(relocs));
|
|
|
|
|
|
|
|
if (bcm1250_m3_war()) {
|
2010-03-23 10:56:38 -06:00
|
|
|
unsigned int segbits = 44;
|
|
|
|
|
|
|
|
uasm_i_dmfc0(&p, K0, C0_BADVADDR);
|
|
|
|
uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_xor(&p, K0, K0, K1);
|
2010-04-28 13:16:17 -06:00
|
|
|
uasm_i_dsrl_safe(&p, K1, K0, 62);
|
|
|
|
uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
|
|
|
|
uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
|
2010-03-23 10:56:38 -06:00
|
|
|
uasm_i_or(&p, K0, K0, K1);
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_il_bnez(&p, &r, K0, label_leave);
|
|
|
|
/* No need for uasm_i_nop */
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2011-07-05 17:34:46 -06:00
|
|
|
wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
|
|
|
|
build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
|
2006-08-23 07:26:50 -06:00
|
|
|
if (m4kc_tlbp_war())
|
|
|
|
build_tlb_probe_entry(&p);
|
2010-02-10 16:12:47 -07:00
|
|
|
|
2014-07-15 07:09:56 -06:00
|
|
|
if (cpu_has_rixi && !cpu_has_rixiex) {
|
2010-02-10 16:12:47 -07:00
|
|
|
/*
|
|
|
|
* If the page is not _PAGE_VALID, RI or XI could not
|
|
|
|
* have triggered it. Skip the expensive test..
|
|
|
|
*/
|
2010-12-20 16:54:50 -07:00
|
|
|
if (use_bbit_insns()) {
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
|
2010-12-20 16:54:50 -07:00
|
|
|
label_tlbl_goaround1);
|
|
|
|
} else {
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
|
|
|
|
uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1);
|
2010-12-20 16:54:50 -07:00
|
|
|
}
|
2010-02-10 16:12:47 -07:00
|
|
|
uasm_i_nop(&p);
|
|
|
|
|
|
|
|
uasm_i_tlbr(&p);
|
2013-06-20 06:56:17 -06:00
|
|
|
|
|
|
|
switch (current_cpu_type()) {
|
|
|
|
default:
|
2014-11-24 08:42:46 -07:00
|
|
|
if (cpu_has_mips_r2_exec_hazard) {
|
2013-06-20 06:56:17 -06:00
|
|
|
uasm_i_ehb(&p);
|
|
|
|
|
|
|
|
case CPU_CAVIUM_OCTEON:
|
|
|
|
case CPU_CAVIUM_OCTEON_PLUS:
|
|
|
|
case CPU_CAVIUM_OCTEON2:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-02-10 16:12:47 -07:00
|
|
|
/* Examine entrylo 0 or 1 based on ptr. */
|
2010-12-20 16:54:50 -07:00
|
|
|
if (use_bbit_insns()) {
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
|
2010-12-20 16:54:50 -07:00
|
|
|
} else {
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
|
|
|
|
uasm_i_beqz(&p, wr.r3, 8);
|
2010-12-20 16:54:50 -07:00
|
|
|
}
|
2011-07-05 17:34:46 -06:00
|
|
|
/* load it in the delay slot*/
|
|
|
|
UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
|
|
|
|
/* load it if ptr is odd */
|
|
|
|
UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
|
2010-02-10 16:12:47 -07:00
|
|
|
/*
|
2011-07-05 17:34:46 -06:00
|
|
|
* If the entryLo (now in wr.r3) is valid (bit 1), RI or
|
2010-02-10 16:12:47 -07:00
|
|
|
* XI must have triggered it.
|
|
|
|
*/
|
2010-12-20 16:54:50 -07:00
|
|
|
if (use_bbit_insns()) {
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl);
|
|
|
|
uasm_i_nop(&p);
|
2010-12-20 16:54:50 -07:00
|
|
|
uasm_l_tlbl_goaround1(&l, p);
|
|
|
|
} else {
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_i_andi(&p, wr.r3, wr.r3, 2);
|
|
|
|
uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl);
|
|
|
|
uasm_i_nop(&p);
|
2010-12-20 16:54:50 -07:00
|
|
|
}
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_l_tlbl_goaround1(&l, p);
|
2010-02-10 16:12:47 -07:00
|
|
|
}
|
2016-04-19 02:25:07 -06:00
|
|
|
build_make_valid(&p, &r, wr.r1, wr.r2, wr.r3);
|
2011-07-05 17:34:46 -06:00
|
|
|
build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-10-16 16:48:10 -06:00
|
|
|
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
|
2009-05-27 18:47:44 -06:00
|
|
|
/*
|
|
|
|
* This is the entry point when build_r4000_tlbchange_handler_head
|
|
|
|
* spots a huge page.
|
|
|
|
*/
|
|
|
|
uasm_l_tlb_huge_update(&l, p);
|
2011-07-05 17:34:46 -06:00
|
|
|
iPTE_LW(&p, wr.r1, wr.r2);
|
|
|
|
build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
|
2009-05-27 18:47:44 -06:00
|
|
|
build_tlb_probe_entry(&p);
|
2010-02-10 16:12:47 -07:00
|
|
|
|
2014-07-15 07:09:56 -06:00
|
|
|
if (cpu_has_rixi && !cpu_has_rixiex) {
|
2010-02-10 16:12:47 -07:00
|
|
|
/*
|
|
|
|
* If the page is not _PAGE_VALID, RI or XI could not
|
|
|
|
* have triggered it. Skip the expensive test..
|
|
|
|
*/
|
2010-12-20 16:54:50 -07:00
|
|
|
if (use_bbit_insns()) {
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
|
2010-12-20 16:54:50 -07:00
|
|
|
label_tlbl_goaround2);
|
|
|
|
} else {
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
|
|
|
|
uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
|
2010-12-20 16:54:50 -07:00
|
|
|
}
|
2010-02-10 16:12:47 -07:00
|
|
|
uasm_i_nop(&p);
|
|
|
|
|
|
|
|
uasm_i_tlbr(&p);
|
2013-06-20 06:56:17 -06:00
|
|
|
|
|
|
|
switch (current_cpu_type()) {
|
|
|
|
default:
|
2014-11-24 08:42:46 -07:00
|
|
|
if (cpu_has_mips_r2_exec_hazard) {
|
2013-06-20 06:56:17 -06:00
|
|
|
uasm_i_ehb(&p);
|
|
|
|
|
|
|
|
case CPU_CAVIUM_OCTEON:
|
|
|
|
case CPU_CAVIUM_OCTEON_PLUS:
|
|
|
|
case CPU_CAVIUM_OCTEON2:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-02-10 16:12:47 -07:00
|
|
|
/* Examine entrylo 0 or 1 based on ptr. */
|
2010-12-20 16:54:50 -07:00
|
|
|
if (use_bbit_insns()) {
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
|
2010-12-20 16:54:50 -07:00
|
|
|
} else {
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
|
|
|
|
uasm_i_beqz(&p, wr.r3, 8);
|
2010-12-20 16:54:50 -07:00
|
|
|
}
|
2011-07-05 17:34:46 -06:00
|
|
|
/* load it in the delay slot*/
|
|
|
|
UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
|
|
|
|
/* load it if ptr is odd */
|
|
|
|
UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
|
2010-02-10 16:12:47 -07:00
|
|
|
/*
|
2011-07-05 17:34:46 -06:00
|
|
|
* If the entryLo (now in wr.r3) is valid (bit 1), RI or
|
2010-02-10 16:12:47 -07:00
|
|
|
* XI must have triggered it.
|
|
|
|
*/
|
2010-12-20 16:54:50 -07:00
|
|
|
if (use_bbit_insns()) {
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2);
|
2010-12-20 16:54:50 -07:00
|
|
|
} else {
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_i_andi(&p, wr.r3, wr.r3, 2);
|
|
|
|
uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
|
2010-12-20 16:54:50 -07:00
|
|
|
}
|
2011-09-16 19:06:02 -06:00
|
|
|
if (PM_DEFAULT_MASK == 0)
|
|
|
|
uasm_i_nop(&p);
|
2010-02-10 16:12:47 -07:00
|
|
|
/*
|
|
|
|
* We clobbered C0_PAGEMASK, restore it. On the other branch
|
|
|
|
* it is restored in build_huge_tlb_write_entry.
|
|
|
|
*/
|
2011-07-05 17:34:46 -06:00
|
|
|
build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0);
|
2010-02-10 16:12:47 -07:00
|
|
|
|
|
|
|
uasm_l_tlbl_goaround2(&l, p);
|
|
|
|
}
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID));
|
|
|
|
build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
|
2009-05-27 18:47:44 -06:00
|
|
|
#endif
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_l_nopage_tlbl(&l, p);
|
2011-07-05 17:34:46 -06:00
|
|
|
build_restore_work_registers(&p);
|
2013-03-25 11:15:55 -06:00
|
|
|
#ifdef CONFIG_CPU_MICROMIPS
|
|
|
|
if ((unsigned long)tlb_do_page_fault_0 & 1) {
|
|
|
|
uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0));
|
|
|
|
uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0));
|
|
|
|
uasm_i_jr(&p, K0);
|
|
|
|
} else
|
|
|
|
#endif
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
|
|
|
|
uasm_i_nop(&p);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
if (p >= handle_tlbl_end)
|
2005-04-16 16:20:36 -06:00
|
|
|
panic("TLB load handler fastpath space exceeded");
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_resolve_relocs(relocs, labels);
|
|
|
|
pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
|
|
|
|
(unsigned int)(p - handle_tlbl));
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_size);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_r4000_tlb_store_handler(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
u32 *p = handle_tlbs;
|
2013-06-23 11:16:19 -06:00
|
|
|
const int handle_tlbs_size = handle_tlbs_end - handle_tlbs;
|
2008-01-28 13:05:38 -07:00
|
|
|
struct uasm_label *l = labels;
|
|
|
|
struct uasm_reloc *r = relocs;
|
2011-07-05 17:34:46 -06:00
|
|
|
struct work_registers wr;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0]));
|
2005-04-16 16:20:36 -06:00
|
|
|
memset(labels, 0, sizeof(labels));
|
|
|
|
memset(relocs, 0, sizeof(relocs));
|
|
|
|
|
2011-07-05 17:34:46 -06:00
|
|
|
wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
|
|
|
|
build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);
|
2006-08-23 07:26:50 -06:00
|
|
|
if (m4kc_tlbp_war())
|
|
|
|
build_tlb_probe_entry(&p);
|
2016-04-19 02:25:07 -06:00
|
|
|
build_make_write(&p, &r, wr.r1, wr.r2, wr.r3);
|
2011-07-05 17:34:46 -06:00
|
|
|
build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-10-16 16:48:10 -06:00
|
|
|
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
|
2009-05-27 18:47:44 -06:00
|
|
|
/*
|
|
|
|
* This is the entry point when
|
|
|
|
* build_r4000_tlbchange_handler_head spots a huge page.
|
|
|
|
*/
|
|
|
|
uasm_l_tlb_huge_update(&l, p);
|
2011-07-05 17:34:46 -06:00
|
|
|
iPTE_LW(&p, wr.r1, wr.r2);
|
|
|
|
build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);
|
2009-05-27 18:47:44 -06:00
|
|
|
build_tlb_probe_entry(&p);
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_i_ori(&p, wr.r1, wr.r1,
|
2009-05-27 18:47:44 -06:00
|
|
|
_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
|
2011-07-05 17:34:46 -06:00
|
|
|
build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
|
2009-05-27 18:47:44 -06:00
|
|
|
#endif
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_l_nopage_tlbs(&l, p);
|
2011-07-05 17:34:46 -06:00
|
|
|
build_restore_work_registers(&p);
|
2013-03-25 11:15:55 -06:00
|
|
|
#ifdef CONFIG_CPU_MICROMIPS
|
|
|
|
if ((unsigned long)tlb_do_page_fault_1 & 1) {
|
|
|
|
uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1));
|
|
|
|
uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1));
|
|
|
|
uasm_i_jr(&p, K0);
|
|
|
|
} else
|
|
|
|
#endif
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
|
|
|
|
uasm_i_nop(&p);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
if (p >= handle_tlbs_end)
|
2005-04-16 16:20:36 -06:00
|
|
|
panic("TLB store handler fastpath space exceeded");
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_resolve_relocs(relocs, labels);
|
|
|
|
pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
|
|
|
|
(unsigned int)(p - handle_tlbs));
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
dump_handler("r4000_tlb_store", handle_tlbs, handle_tlbs_size);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void build_r4000_tlb_modify_handler(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
u32 *p = handle_tlbm;
|
2013-06-23 11:16:19 -06:00
|
|
|
const int handle_tlbm_size = handle_tlbm_end - handle_tlbm;
|
2008-01-28 13:05:38 -07:00
|
|
|
struct uasm_label *l = labels;
|
|
|
|
struct uasm_reloc *r = relocs;
|
2011-07-05 17:34:46 -06:00
|
|
|
struct work_registers wr;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0]));
|
2005-04-16 16:20:36 -06:00
|
|
|
memset(labels, 0, sizeof(labels));
|
|
|
|
memset(relocs, 0, sizeof(relocs));
|
|
|
|
|
2011-07-05 17:34:46 -06:00
|
|
|
wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
|
|
|
|
build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm);
|
2006-08-23 07:26:50 -06:00
|
|
|
if (m4kc_tlbp_war())
|
|
|
|
build_tlb_probe_entry(&p);
|
2005-04-16 16:20:36 -06:00
|
|
|
/* Present and writable bits set, set accessed and dirty bits. */
|
2016-04-19 02:25:07 -06:00
|
|
|
build_make_write(&p, &r, wr.r1, wr.r2, wr.r3);
|
2011-07-05 17:34:46 -06:00
|
|
|
build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-10-16 16:48:10 -06:00
|
|
|
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
|
2009-05-27 18:47:44 -06:00
|
|
|
/*
|
|
|
|
* This is the entry point when
|
|
|
|
* build_r4000_tlbchange_handler_head spots a huge page.
|
|
|
|
*/
|
|
|
|
uasm_l_tlb_huge_update(&l, p);
|
2011-07-05 17:34:46 -06:00
|
|
|
iPTE_LW(&p, wr.r1, wr.r2);
|
|
|
|
build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm);
|
2009-05-27 18:47:44 -06:00
|
|
|
build_tlb_probe_entry(&p);
|
2011-07-05 17:34:46 -06:00
|
|
|
uasm_i_ori(&p, wr.r1, wr.r1,
|
2009-05-27 18:47:44 -06:00
|
|
|
_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
|
2011-07-05 17:34:46 -06:00
|
|
|
build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
|
2009-05-27 18:47:44 -06:00
|
|
|
#endif
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_l_nopage_tlbm(&l, p);
|
2011-07-05 17:34:46 -06:00
|
|
|
build_restore_work_registers(&p);
|
2013-03-25 11:15:55 -06:00
|
|
|
#ifdef CONFIG_CPU_MICROMIPS
|
|
|
|
if ((unsigned long)tlb_do_page_fault_1 & 1) {
|
|
|
|
uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1));
|
|
|
|
uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1));
|
|
|
|
uasm_i_jr(&p, K0);
|
|
|
|
} else
|
|
|
|
#endif
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
|
|
|
|
uasm_i_nop(&p);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
if (p >= handle_tlbm_end)
|
2005-04-16 16:20:36 -06:00
|
|
|
panic("TLB modify handler fastpath space exceeded");
|
|
|
|
|
2008-01-28 13:05:38 -07:00
|
|
|
uasm_resolve_relocs(relocs, labels);
|
|
|
|
pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
|
|
|
|
(unsigned int)(p - handle_tlbm));
|
2006-07-08 18:47:06 -06:00
|
|
|
|
2013-06-23 11:16:19 -06:00
|
|
|
dump_handler("r4000_tlb_modify", handle_tlbm, handle_tlbm_size);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
static void flush_tlb_handlers(void)
|
2013-06-21 11:48:48 -06:00
|
|
|
{
|
|
|
|
local_flush_icache_range((unsigned long)handle_tlbl,
|
2013-07-02 09:19:04 -06:00
|
|
|
(unsigned long)handle_tlbl_end);
|
2013-06-21 11:48:48 -06:00
|
|
|
local_flush_icache_range((unsigned long)handle_tlbs,
|
2013-07-02 09:19:04 -06:00
|
|
|
(unsigned long)handle_tlbs_end);
|
2013-06-21 11:48:48 -06:00
|
|
|
local_flush_icache_range((unsigned long)handle_tlbm,
|
2013-07-02 09:19:04 -06:00
|
|
|
(unsigned long)handle_tlbm_end);
|
|
|
|
local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
|
|
|
|
(unsigned long)tlbmiss_handler_setup_pgd_end);
|
2013-06-21 11:48:48 -06:00
|
|
|
}
|
|
|
|
|
MIPS: mm: Use the Hardware Page Table Walker if the core supports it
The Hardware Page Table Walker aims to speed up TLB refill exceptions
by handling them in the hardware level instead of having a software
TLB refill handler. However, a TLB refill exception can still be
thrown in certain cases such as, synchronus exceptions, or address
translation or memory errors during the HTW operation. As a result of
which, HTW must not be considered a complete replacement for the TLB
refill software handler, but rather a fast-path for it.
For HTW to work, the PWBase register must contain the task's page
global directory address so the HTW will kick in on TLB refill
exceptions.
Due to HTW being a separate engine embedded deep in the CPU pipeline,
we need to restart the HTW everytime a PTE changes to avoid HTW
fetching a old entry from the page tables. It's also necessary to
restart the HTW on context switches to prevent it from fetching a
page from the previous process. Finally, since HTW is using the
entryhi register to write the translations to the TLB, it's necessary
to stop the HTW whenever the entryhi changes (eg for tlb probe
perations) and enable it back afterwards.
== Performance ==
The following trivial test was used to measure the performance of the
HTW. Using the same root filesystem, the following command was used
to measure the number of tlb refill handler executions with and
without (using 'nohtw' kernel parameter) HTW support. The kernel was
modified to use a scratch register as a counter for the TLB refill
exceptions.
find /usr -type f -exec ls -lh {} \;
HTW Enabled:
TLB refill exceptions: 12306
HTW Disabled:
TLB refill exceptions: 17805
Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>
Cc: linux-mips@linux-mips.org
Cc: Markos Chandras <markos.chandras@imgtec.com>
Patchwork: https://patchwork.linux-mips.org/patch/7336/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2014-07-14 05:47:09 -06:00
|
|
|
static void print_htw_config(void)
|
|
|
|
{
|
|
|
|
unsigned long config;
|
|
|
|
unsigned int pwctl;
|
|
|
|
const int field = 2 * sizeof(unsigned long);
|
|
|
|
|
|
|
|
config = read_c0_pwfield();
|
|
|
|
pr_debug("PWField (0x%0*lx): GDI: 0x%02lx UDI: 0x%02lx MDI: 0x%02lx PTI: 0x%02lx PTEI: 0x%02lx\n",
|
|
|
|
field, config,
|
|
|
|
(config & MIPS_PWFIELD_GDI_MASK) >> MIPS_PWFIELD_GDI_SHIFT,
|
|
|
|
(config & MIPS_PWFIELD_UDI_MASK) >> MIPS_PWFIELD_UDI_SHIFT,
|
|
|
|
(config & MIPS_PWFIELD_MDI_MASK) >> MIPS_PWFIELD_MDI_SHIFT,
|
|
|
|
(config & MIPS_PWFIELD_PTI_MASK) >> MIPS_PWFIELD_PTI_SHIFT,
|
|
|
|
(config & MIPS_PWFIELD_PTEI_MASK) >> MIPS_PWFIELD_PTEI_SHIFT);
|
|
|
|
|
|
|
|
config = read_c0_pwsize();
|
2016-05-27 15:25:22 -06:00
|
|
|
pr_debug("PWSize (0x%0*lx): PS: 0x%lx GDW: 0x%02lx UDW: 0x%02lx MDW: 0x%02lx PTW: 0x%02lx PTEW: 0x%02lx\n",
|
MIPS: mm: Use the Hardware Page Table Walker if the core supports it
The Hardware Page Table Walker aims to speed up TLB refill exceptions
by handling them in the hardware level instead of having a software
TLB refill handler. However, a TLB refill exception can still be
thrown in certain cases such as, synchronus exceptions, or address
translation or memory errors during the HTW operation. As a result of
which, HTW must not be considered a complete replacement for the TLB
refill software handler, but rather a fast-path for it.
For HTW to work, the PWBase register must contain the task's page
global directory address so the HTW will kick in on TLB refill
exceptions.
Due to HTW being a separate engine embedded deep in the CPU pipeline,
we need to restart the HTW everytime a PTE changes to avoid HTW
fetching a old entry from the page tables. It's also necessary to
restart the HTW on context switches to prevent it from fetching a
page from the previous process. Finally, since HTW is using the
entryhi register to write the translations to the TLB, it's necessary
to stop the HTW whenever the entryhi changes (eg for tlb probe
perations) and enable it back afterwards.
== Performance ==
The following trivial test was used to measure the performance of the
HTW. Using the same root filesystem, the following command was used
to measure the number of tlb refill handler executions with and
without (using 'nohtw' kernel parameter) HTW support. The kernel was
modified to use a scratch register as a counter for the TLB refill
exceptions.
find /usr -type f -exec ls -lh {} \;
HTW Enabled:
TLB refill exceptions: 12306
HTW Disabled:
TLB refill exceptions: 17805
Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>
Cc: linux-mips@linux-mips.org
Cc: Markos Chandras <markos.chandras@imgtec.com>
Patchwork: https://patchwork.linux-mips.org/patch/7336/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2014-07-14 05:47:09 -06:00
|
|
|
field, config,
|
2016-05-27 15:25:22 -06:00
|
|
|
(config & MIPS_PWSIZE_PS_MASK) >> MIPS_PWSIZE_PS_SHIFT,
|
MIPS: mm: Use the Hardware Page Table Walker if the core supports it
The Hardware Page Table Walker aims to speed up TLB refill exceptions
by handling them in the hardware level instead of having a software
TLB refill handler. However, a TLB refill exception can still be
thrown in certain cases such as, synchronus exceptions, or address
translation or memory errors during the HTW operation. As a result of
which, HTW must not be considered a complete replacement for the TLB
refill software handler, but rather a fast-path for it.
For HTW to work, the PWBase register must contain the task's page
global directory address so the HTW will kick in on TLB refill
exceptions.
Due to HTW being a separate engine embedded deep in the CPU pipeline,
we need to restart the HTW everytime a PTE changes to avoid HTW
fetching a old entry from the page tables. It's also necessary to
restart the HTW on context switches to prevent it from fetching a
page from the previous process. Finally, since HTW is using the
entryhi register to write the translations to the TLB, it's necessary
to stop the HTW whenever the entryhi changes (eg for tlb probe
perations) and enable it back afterwards.
== Performance ==
The following trivial test was used to measure the performance of the
HTW. Using the same root filesystem, the following command was used
to measure the number of tlb refill handler executions with and
without (using 'nohtw' kernel parameter) HTW support. The kernel was
modified to use a scratch register as a counter for the TLB refill
exceptions.
find /usr -type f -exec ls -lh {} \;
HTW Enabled:
TLB refill exceptions: 12306
HTW Disabled:
TLB refill exceptions: 17805
Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>
Cc: linux-mips@linux-mips.org
Cc: Markos Chandras <markos.chandras@imgtec.com>
Patchwork: https://patchwork.linux-mips.org/patch/7336/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2014-07-14 05:47:09 -06:00
|
|
|
(config & MIPS_PWSIZE_GDW_MASK) >> MIPS_PWSIZE_GDW_SHIFT,
|
|
|
|
(config & MIPS_PWSIZE_UDW_MASK) >> MIPS_PWSIZE_UDW_SHIFT,
|
|
|
|
(config & MIPS_PWSIZE_MDW_MASK) >> MIPS_PWSIZE_MDW_SHIFT,
|
|
|
|
(config & MIPS_PWSIZE_PTW_MASK) >> MIPS_PWSIZE_PTW_SHIFT,
|
|
|
|
(config & MIPS_PWSIZE_PTEW_MASK) >> MIPS_PWSIZE_PTEW_SHIFT);
|
|
|
|
|
|
|
|
pwctl = read_c0_pwctl();
|
2016-05-27 15:25:22 -06:00
|
|
|
pr_debug("PWCtl (0x%x): PWEn: 0x%x XK: 0x%x XS: 0x%x XU: 0x%x DPH: 0x%x HugePg: 0x%x Psn: 0x%x\n",
|
MIPS: mm: Use the Hardware Page Table Walker if the core supports it
The Hardware Page Table Walker aims to speed up TLB refill exceptions
by handling them in the hardware level instead of having a software
TLB refill handler. However, a TLB refill exception can still be
thrown in certain cases such as, synchronus exceptions, or address
translation or memory errors during the HTW operation. As a result of
which, HTW must not be considered a complete replacement for the TLB
refill software handler, but rather a fast-path for it.
For HTW to work, the PWBase register must contain the task's page
global directory address so the HTW will kick in on TLB refill
exceptions.
Due to HTW being a separate engine embedded deep in the CPU pipeline,
we need to restart the HTW everytime a PTE changes to avoid HTW
fetching a old entry from the page tables. It's also necessary to
restart the HTW on context switches to prevent it from fetching a
page from the previous process. Finally, since HTW is using the
entryhi register to write the translations to the TLB, it's necessary
to stop the HTW whenever the entryhi changes (eg for tlb probe
perations) and enable it back afterwards.
== Performance ==
The following trivial test was used to measure the performance of the
HTW. Using the same root filesystem, the following command was used
to measure the number of tlb refill handler executions with and
without (using 'nohtw' kernel parameter) HTW support. The kernel was
modified to use a scratch register as a counter for the TLB refill
exceptions.
find /usr -type f -exec ls -lh {} \;
HTW Enabled:
TLB refill exceptions: 12306
HTW Disabled:
TLB refill exceptions: 17805
Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>
Cc: linux-mips@linux-mips.org
Cc: Markos Chandras <markos.chandras@imgtec.com>
Patchwork: https://patchwork.linux-mips.org/patch/7336/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2014-07-14 05:47:09 -06:00
|
|
|
pwctl,
|
|
|
|
(pwctl & MIPS_PWCTL_PWEN_MASK) >> MIPS_PWCTL_PWEN_SHIFT,
|
2016-05-27 15:25:22 -06:00
|
|
|
(pwctl & MIPS_PWCTL_XK_MASK) >> MIPS_PWCTL_XK_SHIFT,
|
|
|
|
(pwctl & MIPS_PWCTL_XS_MASK) >> MIPS_PWCTL_XS_SHIFT,
|
|
|
|
(pwctl & MIPS_PWCTL_XU_MASK) >> MIPS_PWCTL_XU_SHIFT,
|
MIPS: mm: Use the Hardware Page Table Walker if the core supports it
The Hardware Page Table Walker aims to speed up TLB refill exceptions
by handling them in the hardware level instead of having a software
TLB refill handler. However, a TLB refill exception can still be
thrown in certain cases such as, synchronus exceptions, or address
translation or memory errors during the HTW operation. As a result of
which, HTW must not be considered a complete replacement for the TLB
refill software handler, but rather a fast-path for it.
For HTW to work, the PWBase register must contain the task's page
global directory address so the HTW will kick in on TLB refill
exceptions.
Due to HTW being a separate engine embedded deep in the CPU pipeline,
we need to restart the HTW everytime a PTE changes to avoid HTW
fetching a old entry from the page tables. It's also necessary to
restart the HTW on context switches to prevent it from fetching a
page from the previous process. Finally, since HTW is using the
entryhi register to write the translations to the TLB, it's necessary
to stop the HTW whenever the entryhi changes (eg for tlb probe
perations) and enable it back afterwards.
== Performance ==
The following trivial test was used to measure the performance of the
HTW. Using the same root filesystem, the following command was used
to measure the number of tlb refill handler executions with and
without (using 'nohtw' kernel parameter) HTW support. The kernel was
modified to use a scratch register as a counter for the TLB refill
exceptions.
find /usr -type f -exec ls -lh {} \;
HTW Enabled:
TLB refill exceptions: 12306
HTW Disabled:
TLB refill exceptions: 17805
Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>
Cc: linux-mips@linux-mips.org
Cc: Markos Chandras <markos.chandras@imgtec.com>
Patchwork: https://patchwork.linux-mips.org/patch/7336/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2014-07-14 05:47:09 -06:00
|
|
|
(pwctl & MIPS_PWCTL_DPH_MASK) >> MIPS_PWCTL_DPH_SHIFT,
|
|
|
|
(pwctl & MIPS_PWCTL_HUGEPG_MASK) >> MIPS_PWCTL_HUGEPG_SHIFT,
|
|
|
|
(pwctl & MIPS_PWCTL_PSN_MASK) >> MIPS_PWCTL_PSN_SHIFT);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void config_htw_params(void)
|
|
|
|
{
|
|
|
|
unsigned long pwfield, pwsize, ptei;
|
|
|
|
unsigned int config;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We are using 2-level page tables, so we only need to
|
|
|
|
* setup GDW and PTW appropriately. UDW and MDW will remain 0.
|
|
|
|
* The default value of GDI/UDI/MDI/PTI is 0xc. It is illegal to
|
|
|
|
* write values less than 0xc in these fields because the entire
|
|
|
|
* write will be dropped. As a result of which, we must preserve
|
|
|
|
* the original reset values and overwrite only what we really want.
|
|
|
|
*/
|
|
|
|
|
|
|
|
pwfield = read_c0_pwfield();
|
|
|
|
/* re-initialize the GDI field */
|
|
|
|
pwfield &= ~MIPS_PWFIELD_GDI_MASK;
|
|
|
|
pwfield |= PGDIR_SHIFT << MIPS_PWFIELD_GDI_SHIFT;
|
|
|
|
/* re-initialize the PTI field including the even/odd bit */
|
|
|
|
pwfield &= ~MIPS_PWFIELD_PTI_MASK;
|
|
|
|
pwfield |= PAGE_SHIFT << MIPS_PWFIELD_PTI_SHIFT;
|
2015-09-22 13:03:37 -06:00
|
|
|
if (CONFIG_PGTABLE_LEVELS >= 3) {
|
|
|
|
pwfield &= ~MIPS_PWFIELD_MDI_MASK;
|
|
|
|
pwfield |= PMD_SHIFT << MIPS_PWFIELD_MDI_SHIFT;
|
|
|
|
}
|
MIPS: mm: Use the Hardware Page Table Walker if the core supports it
The Hardware Page Table Walker aims to speed up TLB refill exceptions
by handling them in the hardware level instead of having a software
TLB refill handler. However, a TLB refill exception can still be
thrown in certain cases such as, synchronus exceptions, or address
translation or memory errors during the HTW operation. As a result of
which, HTW must not be considered a complete replacement for the TLB
refill software handler, but rather a fast-path for it.
For HTW to work, the PWBase register must contain the task's page
global directory address so the HTW will kick in on TLB refill
exceptions.
Due to HTW being a separate engine embedded deep in the CPU pipeline,
we need to restart the HTW everytime a PTE changes to avoid HTW
fetching a old entry from the page tables. It's also necessary to
restart the HTW on context switches to prevent it from fetching a
page from the previous process. Finally, since HTW is using the
entryhi register to write the translations to the TLB, it's necessary
to stop the HTW whenever the entryhi changes (eg for tlb probe
perations) and enable it back afterwards.
== Performance ==
The following trivial test was used to measure the performance of the
HTW. Using the same root filesystem, the following command was used
to measure the number of tlb refill handler executions with and
without (using 'nohtw' kernel parameter) HTW support. The kernel was
modified to use a scratch register as a counter for the TLB refill
exceptions.
find /usr -type f -exec ls -lh {} \;
HTW Enabled:
TLB refill exceptions: 12306
HTW Disabled:
TLB refill exceptions: 17805
Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>
Cc: linux-mips@linux-mips.org
Cc: Markos Chandras <markos.chandras@imgtec.com>
Patchwork: https://patchwork.linux-mips.org/patch/7336/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2014-07-14 05:47:09 -06:00
|
|
|
/* Set the PTEI right shift */
|
|
|
|
ptei = _PAGE_GLOBAL_SHIFT << MIPS_PWFIELD_PTEI_SHIFT;
|
|
|
|
pwfield |= ptei;
|
|
|
|
write_c0_pwfield(pwfield);
|
|
|
|
/* Check whether the PTEI value is supported */
|
|
|
|
back_to_back_c0_hazard();
|
|
|
|
pwfield = read_c0_pwfield();
|
|
|
|
if (((pwfield & MIPS_PWFIELD_PTEI_MASK) << MIPS_PWFIELD_PTEI_SHIFT)
|
|
|
|
!= ptei) {
|
|
|
|
pr_warn("Unsupported PTEI field value: 0x%lx. HTW will not be enabled",
|
|
|
|
ptei);
|
|
|
|
/*
|
|
|
|
* Drop option to avoid HTW being enabled via another path
|
|
|
|
* (eg htw_reset())
|
|
|
|
*/
|
|
|
|
current_cpu_data.options &= ~MIPS_CPU_HTW;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
pwsize = ilog2(PTRS_PER_PGD) << MIPS_PWSIZE_GDW_SHIFT;
|
|
|
|
pwsize |= ilog2(PTRS_PER_PTE) << MIPS_PWSIZE_PTW_SHIFT;
|
2015-09-22 13:03:37 -06:00
|
|
|
if (CONFIG_PGTABLE_LEVELS >= 3)
|
|
|
|
pwsize |= ilog2(PTRS_PER_PMD) << MIPS_PWSIZE_MDW_SHIFT;
|
2015-02-26 17:16:38 -07:00
|
|
|
|
2016-05-27 15:25:23 -06:00
|
|
|
/* Set pointer size to size of directory pointers */
|
|
|
|
if (config_enabled(CONFIG_64BIT))
|
|
|
|
pwsize |= MIPS_PWSIZE_PS_MASK;
|
|
|
|
/* PTEs may be multiple pointers long (e.g. with XPA) */
|
|
|
|
pwsize |= ((PTE_T_LOG2 - PGD_T_LOG2) << MIPS_PWSIZE_PTEW_SHIFT)
|
|
|
|
& MIPS_PWSIZE_PTEW_MASK;
|
2015-02-26 17:16:38 -07:00
|
|
|
|
MIPS: mm: Use the Hardware Page Table Walker if the core supports it
The Hardware Page Table Walker aims to speed up TLB refill exceptions
by handling them in the hardware level instead of having a software
TLB refill handler. However, a TLB refill exception can still be
thrown in certain cases such as, synchronus exceptions, or address
translation or memory errors during the HTW operation. As a result of
which, HTW must not be considered a complete replacement for the TLB
refill software handler, but rather a fast-path for it.
For HTW to work, the PWBase register must contain the task's page
global directory address so the HTW will kick in on TLB refill
exceptions.
Due to HTW being a separate engine embedded deep in the CPU pipeline,
we need to restart the HTW everytime a PTE changes to avoid HTW
fetching a old entry from the page tables. It's also necessary to
restart the HTW on context switches to prevent it from fetching a
page from the previous process. Finally, since HTW is using the
entryhi register to write the translations to the TLB, it's necessary
to stop the HTW whenever the entryhi changes (eg for tlb probe
perations) and enable it back afterwards.
== Performance ==
The following trivial test was used to measure the performance of the
HTW. Using the same root filesystem, the following command was used
to measure the number of tlb refill handler executions with and
without (using 'nohtw' kernel parameter) HTW support. The kernel was
modified to use a scratch register as a counter for the TLB refill
exceptions.
find /usr -type f -exec ls -lh {} \;
HTW Enabled:
TLB refill exceptions: 12306
HTW Disabled:
TLB refill exceptions: 17805
Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>
Cc: linux-mips@linux-mips.org
Cc: Markos Chandras <markos.chandras@imgtec.com>
Patchwork: https://patchwork.linux-mips.org/patch/7336/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2014-07-14 05:47:09 -06:00
|
|
|
write_c0_pwsize(pwsize);
|
|
|
|
|
|
|
|
/* Make sure everything is set before we enable the HTW */
|
|
|
|
back_to_back_c0_hazard();
|
|
|
|
|
2016-05-27 15:25:23 -06:00
|
|
|
/*
|
|
|
|
* Enable HTW (and only for XUSeg on 64-bit), and disable the rest of
|
|
|
|
* the pwctl fields.
|
|
|
|
*/
|
MIPS: mm: Use the Hardware Page Table Walker if the core supports it
The Hardware Page Table Walker aims to speed up TLB refill exceptions
by handling them in the hardware level instead of having a software
TLB refill handler. However, a TLB refill exception can still be
thrown in certain cases such as, synchronus exceptions, or address
translation or memory errors during the HTW operation. As a result of
which, HTW must not be considered a complete replacement for the TLB
refill software handler, but rather a fast-path for it.
For HTW to work, the PWBase register must contain the task's page
global directory address so the HTW will kick in on TLB refill
exceptions.
Due to HTW being a separate engine embedded deep in the CPU pipeline,
we need to restart the HTW everytime a PTE changes to avoid HTW
fetching a old entry from the page tables. It's also necessary to
restart the HTW on context switches to prevent it from fetching a
page from the previous process. Finally, since HTW is using the
entryhi register to write the translations to the TLB, it's necessary
to stop the HTW whenever the entryhi changes (eg for tlb probe
perations) and enable it back afterwards.
== Performance ==
The following trivial test was used to measure the performance of the
HTW. Using the same root filesystem, the following command was used
to measure the number of tlb refill handler executions with and
without (using 'nohtw' kernel parameter) HTW support. The kernel was
modified to use a scratch register as a counter for the TLB refill
exceptions.
find /usr -type f -exec ls -lh {} \;
HTW Enabled:
TLB refill exceptions: 12306
HTW Disabled:
TLB refill exceptions: 17805
Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>
Cc: linux-mips@linux-mips.org
Cc: Markos Chandras <markos.chandras@imgtec.com>
Patchwork: https://patchwork.linux-mips.org/patch/7336/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2014-07-14 05:47:09 -06:00
|
|
|
config = 1 << MIPS_PWCTL_PWEN_SHIFT;
|
2016-05-27 15:25:23 -06:00
|
|
|
if (config_enabled(CONFIG_64BIT))
|
|
|
|
config |= MIPS_PWCTL_XU_MASK;
|
MIPS: mm: Use the Hardware Page Table Walker if the core supports it
The Hardware Page Table Walker aims to speed up TLB refill exceptions
by handling them in the hardware level instead of having a software
TLB refill handler. However, a TLB refill exception can still be
thrown in certain cases such as, synchronus exceptions, or address
translation or memory errors during the HTW operation. As a result of
which, HTW must not be considered a complete replacement for the TLB
refill software handler, but rather a fast-path for it.
For HTW to work, the PWBase register must contain the task's page
global directory address so the HTW will kick in on TLB refill
exceptions.
Due to HTW being a separate engine embedded deep in the CPU pipeline,
we need to restart the HTW everytime a PTE changes to avoid HTW
fetching a old entry from the page tables. It's also necessary to
restart the HTW on context switches to prevent it from fetching a
page from the previous process. Finally, since HTW is using the
entryhi register to write the translations to the TLB, it's necessary
to stop the HTW whenever the entryhi changes (eg for tlb probe
perations) and enable it back afterwards.
== Performance ==
The following trivial test was used to measure the performance of the
HTW. Using the same root filesystem, the following command was used
to measure the number of tlb refill handler executions with and
without (using 'nohtw' kernel parameter) HTW support. The kernel was
modified to use a scratch register as a counter for the TLB refill
exceptions.
find /usr -type f -exec ls -lh {} \;
HTW Enabled:
TLB refill exceptions: 12306
HTW Disabled:
TLB refill exceptions: 17805
Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>
Cc: linux-mips@linux-mips.org
Cc: Markos Chandras <markos.chandras@imgtec.com>
Patchwork: https://patchwork.linux-mips.org/patch/7336/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2014-07-14 05:47:09 -06:00
|
|
|
write_c0_pwctl(config);
|
|
|
|
pr_info("Hardware Page Table Walker enabled\n");
|
|
|
|
|
|
|
|
print_htw_config();
|
|
|
|
}
|
|
|
|
|
2015-02-26 17:16:38 -07:00
|
|
|
static void config_xpa_params(void)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_XPA
|
|
|
|
unsigned int pagegrain;
|
|
|
|
|
|
|
|
if (mips_xpa_disabled) {
|
|
|
|
pr_info("Extended Physical Addressing (XPA) disabled\n");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
pagegrain = read_c0_pagegrain();
|
|
|
|
write_c0_pagegrain(pagegrain | PG_ELPA);
|
|
|
|
back_to_back_c0_hazard();
|
|
|
|
pagegrain = read_c0_pagegrain();
|
|
|
|
|
|
|
|
if (pagegrain & PG_ELPA)
|
|
|
|
pr_info("Extended Physical Addressing (XPA) enabled\n");
|
|
|
|
else
|
|
|
|
panic("Extended Physical Addressing (XPA) disabled");
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2015-09-22 12:42:52 -06:00
|
|
|
static void check_pabits(void)
|
|
|
|
{
|
|
|
|
unsigned long entry;
|
|
|
|
unsigned pabits, fillbits;
|
|
|
|
|
|
|
|
if (!cpu_has_rixi || !_PAGE_NO_EXEC) {
|
|
|
|
/*
|
|
|
|
* We'll only be making use of the fact that we can rotate bits
|
|
|
|
* into the fill if the CPU supports RIXI, so don't bother
|
|
|
|
* probing this for CPUs which don't.
|
|
|
|
*/
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
write_c0_entrylo0(~0ul);
|
|
|
|
back_to_back_c0_hazard();
|
|
|
|
entry = read_c0_entrylo0();
|
|
|
|
|
|
|
|
/* clear all non-PFN bits */
|
|
|
|
entry &= ~((1 << MIPS_ENTRYLO_PFN_SHIFT) - 1);
|
|
|
|
entry &= ~(MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI);
|
|
|
|
|
|
|
|
/* find a lower bound on PABITS, and upper bound on fill bits */
|
|
|
|
pabits = fls_long(entry) + 6;
|
|
|
|
fillbits = max_t(int, (int)BITS_PER_LONG - pabits, 0);
|
|
|
|
|
|
|
|
/* minus the RI & XI bits */
|
|
|
|
fillbits -= min_t(unsigned, fillbits, 2);
|
|
|
|
|
|
|
|
if (fillbits >= ilog2(_PAGE_NO_EXEC))
|
|
|
|
fill_includes_sw_bits = true;
|
|
|
|
|
|
|
|
pr_debug("Entry* registers contain %u fill bits\n", fillbits);
|
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 07:38:59 -06:00
|
|
|
void build_tlb_refill_handler(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* The refill handler is generated per-CPU, multi-node systems
|
|
|
|
* may have local storage for it. The other handlers are only
|
|
|
|
* needed once.
|
|
|
|
*/
|
|
|
|
static int run_once = 0;
|
|
|
|
|
2016-04-19 02:25:11 -06:00
|
|
|
if (config_enabled(CONFIG_XPA) && !cpu_has_rixi)
|
|
|
|
panic("Kernels supporting XPA currently require CPUs with RIXI");
|
|
|
|
|
2012-10-16 14:20:26 -06:00
|
|
|
output_pgtable_bits_defines();
|
2015-09-22 12:42:52 -06:00
|
|
|
check_pabits();
|
2012-10-16 14:20:26 -06:00
|
|
|
|
2010-04-28 13:16:18 -06:00
|
|
|
#ifdef CONFIG_64BIT
|
|
|
|
check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
|
|
|
|
#endif
|
|
|
|
|
2007-10-11 16:46:15 -06:00
|
|
|
switch (current_cpu_type()) {
|
2005-04-16 16:20:36 -06:00
|
|
|
case CPU_R2000:
|
|
|
|
case CPU_R3000:
|
|
|
|
case CPU_R3000A:
|
|
|
|
case CPU_R3081E:
|
|
|
|
case CPU_TX3912:
|
|
|
|
case CPU_TX3922:
|
|
|
|
case CPU_TX3927:
|
2009-10-14 13:16:56 -06:00
|
|
|
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
|
MIPS: Build uasm-generated code only once to avoid CPU Hotplug problem
This and the next patch resolve memory corruption problems while CPU
hotplug. Without these patches, memory corruption can triggered easily
as below:
On a quad-core MIPS platform, use "spawn" of UnixBench-5.1.3 (http://
code.google.com/p/byte-unixbench/) and a CPU hotplug script like this
(hotplug.sh):
while true; do
echo 0 >/sys/devices/system/cpu/cpu1/online
echo 0 >/sys/devices/system/cpu/cpu2/online
echo 0 >/sys/devices/system/cpu/cpu3/online
sleep 1
echo 1 >/sys/devices/system/cpu/cpu1/online
echo 1 >/sys/devices/system/cpu/cpu2/online
echo 1 >/sys/devices/system/cpu/cpu3/online
sleep 1
done
Run "hotplug.sh" and then run "spawn 10000", spawn will get segfault
after a few minutes.
This patch:
Currently, clear_page()/copy_page() are generated by Micro-assembler
dynamically. But they are unavailable until uasm_resolve_relocs() has
finished because jump labels are illegal before that. Since these
functions are shared by every CPU, we only call build_clear_page()/
build_copy_page() only once at boot time. Without this patch, programs
will get random memory corruption (segmentation fault, bus error, etc.)
while CPU Hotplug (e.g. one CPU is using clear_page() while another is
generating it in cpu_cache_init()).
For similar reasons we modify build_tlb_refill_handler()'s invocation.
V2:
1, Rework the code to make CPU#0 can be online/offline.
2, Introduce cpu_has_local_ebase feature since some types of MIPS CPU
need a per-CPU tlb_refill_handler().
Signed-off-by: Huacai Chen <chenhc@lemote.com>
Signed-off-by: Hongbing Hu <huhb@lemote.com>
Acked-by: David Daney <david.daney@cavium.com>
Patchwork: http://patchwork.linux-mips.org/patch/4994/
Acked-by: John Crispin <blogic@openwrt.org>
2013-03-17 05:49:38 -06:00
|
|
|
if (cpu_has_local_ebase)
|
|
|
|
build_r3000_tlb_refill_handler();
|
2005-04-16 16:20:36 -06:00
|
|
|
if (!run_once) {
|
MIPS: Build uasm-generated code only once to avoid CPU Hotplug problem
This and the next patch resolve memory corruption problems while CPU
hotplug. Without these patches, memory corruption can triggered easily
as below:
On a quad-core MIPS platform, use "spawn" of UnixBench-5.1.3 (http://
code.google.com/p/byte-unixbench/) and a CPU hotplug script like this
(hotplug.sh):
while true; do
echo 0 >/sys/devices/system/cpu/cpu1/online
echo 0 >/sys/devices/system/cpu/cpu2/online
echo 0 >/sys/devices/system/cpu/cpu3/online
sleep 1
echo 1 >/sys/devices/system/cpu/cpu1/online
echo 1 >/sys/devices/system/cpu/cpu2/online
echo 1 >/sys/devices/system/cpu/cpu3/online
sleep 1
done
Run "hotplug.sh" and then run "spawn 10000", spawn will get segfault
after a few minutes.
This patch:
Currently, clear_page()/copy_page() are generated by Micro-assembler
dynamically. But they are unavailable until uasm_resolve_relocs() has
finished because jump labels are illegal before that. Since these
functions are shared by every CPU, we only call build_clear_page()/
build_copy_page() only once at boot time. Without this patch, programs
will get random memory corruption (segmentation fault, bus error, etc.)
while CPU Hotplug (e.g. one CPU is using clear_page() while another is
generating it in cpu_cache_init()).
For similar reasons we modify build_tlb_refill_handler()'s invocation.
V2:
1, Rework the code to make CPU#0 can be online/offline.
2, Introduce cpu_has_local_ebase feature since some types of MIPS CPU
need a per-CPU tlb_refill_handler().
Signed-off-by: Huacai Chen <chenhc@lemote.com>
Signed-off-by: Hongbing Hu <huhb@lemote.com>
Acked-by: David Daney <david.daney@cavium.com>
Patchwork: http://patchwork.linux-mips.org/patch/4994/
Acked-by: John Crispin <blogic@openwrt.org>
2013-03-17 05:49:38 -06:00
|
|
|
if (!cpu_has_local_ebase)
|
|
|
|
build_r3000_tlb_refill_handler();
|
2013-09-25 04:58:04 -06:00
|
|
|
build_setup_pgd();
|
2005-04-16 16:20:36 -06:00
|
|
|
build_r3000_tlb_load_handler();
|
|
|
|
build_r3000_tlb_store_handler();
|
|
|
|
build_r3000_tlb_modify_handler();
|
2013-06-21 11:48:48 -06:00
|
|
|
flush_tlb_handlers();
|
2005-04-16 16:20:36 -06:00
|
|
|
run_once++;
|
|
|
|
}
|
2009-10-14 13:16:56 -06:00
|
|
|
#else
|
|
|
|
panic("No R3000 TLB refill handler");
|
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
break;
|
|
|
|
|
|
|
|
case CPU_R6000:
|
|
|
|
case CPU_R6000A:
|
|
|
|
panic("No R6000 TLB refill handler yet");
|
|
|
|
break;
|
|
|
|
|
|
|
|
case CPU_R8000:
|
|
|
|
panic("No R8000 TLB refill handler yet");
|
|
|
|
break;
|
|
|
|
|
|
|
|
default:
|
2016-03-02 18:45:12 -07:00
|
|
|
if (cpu_has_ldpte)
|
|
|
|
setup_pw();
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
if (!run_once) {
|
2011-07-05 17:34:46 -06:00
|
|
|
scratch_reg = allocate_kscratch();
|
2013-09-25 04:58:04 -06:00
|
|
|
build_setup_pgd();
|
2005-04-16 16:20:36 -06:00
|
|
|
build_r4000_tlb_load_handler();
|
|
|
|
build_r4000_tlb_store_handler();
|
|
|
|
build_r4000_tlb_modify_handler();
|
2016-03-02 18:45:12 -07:00
|
|
|
if (cpu_has_ldpte)
|
|
|
|
build_loongson3_tlb_refill_handler();
|
|
|
|
else if (!cpu_has_local_ebase)
|
MIPS: Build uasm-generated code only once to avoid CPU Hotplug problem
This and the next patch resolve memory corruption problems while CPU
hotplug. Without these patches, memory corruption can triggered easily
as below:
On a quad-core MIPS platform, use "spawn" of UnixBench-5.1.3 (http://
code.google.com/p/byte-unixbench/) and a CPU hotplug script like this
(hotplug.sh):
while true; do
echo 0 >/sys/devices/system/cpu/cpu1/online
echo 0 >/sys/devices/system/cpu/cpu2/online
echo 0 >/sys/devices/system/cpu/cpu3/online
sleep 1
echo 1 >/sys/devices/system/cpu/cpu1/online
echo 1 >/sys/devices/system/cpu/cpu2/online
echo 1 >/sys/devices/system/cpu/cpu3/online
sleep 1
done
Run "hotplug.sh" and then run "spawn 10000", spawn will get segfault
after a few minutes.
This patch:
Currently, clear_page()/copy_page() are generated by Micro-assembler
dynamically. But they are unavailable until uasm_resolve_relocs() has
finished because jump labels are illegal before that. Since these
functions are shared by every CPU, we only call build_clear_page()/
build_copy_page() only once at boot time. Without this patch, programs
will get random memory corruption (segmentation fault, bus error, etc.)
while CPU Hotplug (e.g. one CPU is using clear_page() while another is
generating it in cpu_cache_init()).
For similar reasons we modify build_tlb_refill_handler()'s invocation.
V2:
1, Rework the code to make CPU#0 can be online/offline.
2, Introduce cpu_has_local_ebase feature since some types of MIPS CPU
need a per-CPU tlb_refill_handler().
Signed-off-by: Huacai Chen <chenhc@lemote.com>
Signed-off-by: Hongbing Hu <huhb@lemote.com>
Acked-by: David Daney <david.daney@cavium.com>
Patchwork: http://patchwork.linux-mips.org/patch/4994/
Acked-by: John Crispin <blogic@openwrt.org>
2013-03-17 05:49:38 -06:00
|
|
|
build_r4000_tlb_refill_handler();
|
2013-06-21 11:48:48 -06:00
|
|
|
flush_tlb_handlers();
|
2005-04-16 16:20:36 -06:00
|
|
|
run_once++;
|
|
|
|
}
|
MIPS: Build uasm-generated code only once to avoid CPU Hotplug problem
This and the next patch resolve memory corruption problems while CPU
hotplug. Without these patches, memory corruption can triggered easily
as below:
On a quad-core MIPS platform, use "spawn" of UnixBench-5.1.3 (http://
code.google.com/p/byte-unixbench/) and a CPU hotplug script like this
(hotplug.sh):
while true; do
echo 0 >/sys/devices/system/cpu/cpu1/online
echo 0 >/sys/devices/system/cpu/cpu2/online
echo 0 >/sys/devices/system/cpu/cpu3/online
sleep 1
echo 1 >/sys/devices/system/cpu/cpu1/online
echo 1 >/sys/devices/system/cpu/cpu2/online
echo 1 >/sys/devices/system/cpu/cpu3/online
sleep 1
done
Run "hotplug.sh" and then run "spawn 10000", spawn will get segfault
after a few minutes.
This patch:
Currently, clear_page()/copy_page() are generated by Micro-assembler
dynamically. But they are unavailable until uasm_resolve_relocs() has
finished because jump labels are illegal before that. Since these
functions are shared by every CPU, we only call build_clear_page()/
build_copy_page() only once at boot time. Without this patch, programs
will get random memory corruption (segmentation fault, bus error, etc.)
while CPU Hotplug (e.g. one CPU is using clear_page() while another is
generating it in cpu_cache_init()).
For similar reasons we modify build_tlb_refill_handler()'s invocation.
V2:
1, Rework the code to make CPU#0 can be online/offline.
2, Introduce cpu_has_local_ebase feature since some types of MIPS CPU
need a per-CPU tlb_refill_handler().
Signed-off-by: Huacai Chen <chenhc@lemote.com>
Signed-off-by: Hongbing Hu <huhb@lemote.com>
Acked-by: David Daney <david.daney@cavium.com>
Patchwork: http://patchwork.linux-mips.org/patch/4994/
Acked-by: John Crispin <blogic@openwrt.org>
2013-03-17 05:49:38 -06:00
|
|
|
if (cpu_has_local_ebase)
|
|
|
|
build_r4000_tlb_refill_handler();
|
2015-02-26 17:16:38 -07:00
|
|
|
if (cpu_has_xpa)
|
|
|
|
config_xpa_params();
|
MIPS: mm: Use the Hardware Page Table Walker if the core supports it
The Hardware Page Table Walker aims to speed up TLB refill exceptions
by handling them in the hardware level instead of having a software
TLB refill handler. However, a TLB refill exception can still be
thrown in certain cases such as, synchronus exceptions, or address
translation or memory errors during the HTW operation. As a result of
which, HTW must not be considered a complete replacement for the TLB
refill software handler, but rather a fast-path for it.
For HTW to work, the PWBase register must contain the task's page
global directory address so the HTW will kick in on TLB refill
exceptions.
Due to HTW being a separate engine embedded deep in the CPU pipeline,
we need to restart the HTW everytime a PTE changes to avoid HTW
fetching a old entry from the page tables. It's also necessary to
restart the HTW on context switches to prevent it from fetching a
page from the previous process. Finally, since HTW is using the
entryhi register to write the translations to the TLB, it's necessary
to stop the HTW whenever the entryhi changes (eg for tlb probe
perations) and enable it back afterwards.
== Performance ==
The following trivial test was used to measure the performance of the
HTW. Using the same root filesystem, the following command was used
to measure the number of tlb refill handler executions with and
without (using 'nohtw' kernel parameter) HTW support. The kernel was
modified to use a scratch register as a counter for the TLB refill
exceptions.
find /usr -type f -exec ls -lh {} \;
HTW Enabled:
TLB refill exceptions: 12306
HTW Disabled:
TLB refill exceptions: 17805
Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>
Cc: linux-mips@linux-mips.org
Cc: Markos Chandras <markos.chandras@imgtec.com>
Patchwork: https://patchwork.linux-mips.org/patch/7336/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2014-07-14 05:47:09 -06:00
|
|
|
if (cpu_has_htw)
|
|
|
|
config_htw_params();
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|