arm64: mm: avoid redundant READ_ONCE(*ptep)
In set_pte_at(), we read the old pte value so that it can be passed into checks for racy hw updates. These checks are only performed for CONFIG_DEBUG_VM, and the value is not used otherwise. Since we read the pte value with READ_ONCE(), the compiler cannot elide the redundant read for !CONFIG_DEBUG_VM kernels. Let's ameliorate matters by moving the read and the checks into a helper, __check_racy_pte_update(), which only performs the read when the value will be used. This also allows us to reformat the conditions for clarity. Acked-by: Will Deacon <will.deacon@arm.com> Signed-off-by: Mark Rutland <mark.rutland@arm.com> Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>alistair/sunxi64-5.4-dsi
parent
4745224b45
commit
9b60472205
|
@ -246,29 +246,42 @@ extern void __sync_icache_dcache(pte_t pteval);
|
|||
*
|
||||
* PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY)
|
||||
*/
|
||||
static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
|
||||
pte_t *ptep, pte_t pte)
|
||||
|
||||
static inline void __check_racy_pte_update(struct mm_struct *mm, pte_t *ptep,
|
||||
pte_t pte)
|
||||
{
|
||||
pte_t old_pte;
|
||||
|
||||
if (!IS_ENABLED(CONFIG_DEBUG_VM))
|
||||
return;
|
||||
|
||||
old_pte = READ_ONCE(*ptep);
|
||||
|
||||
if (!pte_valid(old_pte) || !pte_valid(pte))
|
||||
return;
|
||||
if (mm != current->active_mm && atomic_read(&mm->mm_users) <= 1)
|
||||
return;
|
||||
|
||||
/*
|
||||
* Check for potential race with hardware updates of the pte
|
||||
* (ptep_set_access_flags safely changes valid ptes without going
|
||||
* through an invalid entry).
|
||||
*/
|
||||
VM_WARN_ONCE(!pte_young(pte),
|
||||
"%s: racy access flag clearing: 0x%016llx -> 0x%016llx",
|
||||
__func__, pte_val(old_pte), pte_val(pte));
|
||||
VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte),
|
||||
"%s: racy dirty state clearing: 0x%016llx -> 0x%016llx",
|
||||
__func__, pte_val(old_pte), pte_val(pte));
|
||||
}
|
||||
|
||||
static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
|
||||
pte_t *ptep, pte_t pte)
|
||||
{
|
||||
if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte))
|
||||
__sync_icache_dcache(pte);
|
||||
|
||||
/*
|
||||
* If the existing pte is valid, check for potential race with
|
||||
* hardware updates of the pte (ptep_set_access_flags safely changes
|
||||
* valid ptes without going through an invalid entry).
|
||||
*/
|
||||
old_pte = READ_ONCE(*ptep);
|
||||
if (IS_ENABLED(CONFIG_DEBUG_VM) && pte_valid(old_pte) && pte_valid(pte) &&
|
||||
(mm == current->active_mm || atomic_read(&mm->mm_users) > 1)) {
|
||||
VM_WARN_ONCE(!pte_young(pte),
|
||||
"%s: racy access flag clearing: 0x%016llx -> 0x%016llx",
|
||||
__func__, pte_val(old_pte), pte_val(pte));
|
||||
VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte),
|
||||
"%s: racy dirty state clearing: 0x%016llx -> 0x%016llx",
|
||||
__func__, pte_val(old_pte), pte_val(pte));
|
||||
}
|
||||
__check_racy_pte_update(mm, ptep, pte);
|
||||
|
||||
set_pte(ptep, pte);
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue