mirror of
https://github.com/raspberrypi/linux.git
synced 2026-01-04 18:27:36 +00:00
951c2a51ae
The per CPU hardirq_stack_ptr contains the pointer to the irq stack in the
form that it is ready to be assigned to [ER]SP so that the first push ends
up on the top entry of the stack.
But the stack switching on 64 bit has the following rules:
1) Store the current stack pointer (RSP) in the top most stack entry
to allow the unwinder to link back to the previous stack
2) Set RSP to the top most stack entry
3) Invoke functions on the irq stack
4) Pop RSP from the top most stack entry (stored in #1) so it's back
to the original stack.
That requires all stack switching code to decrement the stored pointer by 8
in order to be able to store the current RSP and then set RSP to that
location. That's a pointless exercise.
Do the -8 adjustment right when storing the pointer and make the data type
a void pointer to avoid confusion vs. the struct irq_stack data type which
is on 64bit only used to declare the backing store. Move the definition
next to the inuse flag so they likely end up in the same cache
line. Sticking them into a struct to enforce it is a seperate change.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Kees Cook <keescook@chromium.org>
Link: https://lore.kernel.org/r/20210210002512.354260928@linutronix.de
107 lines
2.7 KiB
C
107 lines
2.7 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _ASM_X86_IRQ_STACK_H
|
|
#define _ASM_X86_IRQ_STACK_H
|
|
|
|
#include <linux/ptrace.h>
|
|
|
|
#include <asm/processor.h>
|
|
|
|
#ifdef CONFIG_X86_64
|
|
static __always_inline bool irqstack_active(void)
|
|
{
|
|
return __this_cpu_read(hardirq_stack_inuse);
|
|
}
|
|
|
|
void asm_call_on_stack(void *sp, void (*func)(void), void *arg);
|
|
void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs),
|
|
struct pt_regs *regs);
|
|
void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc),
|
|
struct irq_desc *desc);
|
|
|
|
static __always_inline void __run_on_irqstack(void (*func)(void))
|
|
{
|
|
void *tos = __this_cpu_read(hardirq_stack_ptr);
|
|
|
|
__this_cpu_write(hardirq_stack_inuse, true);
|
|
asm_call_on_stack(tos, func, NULL);
|
|
__this_cpu_write(hardirq_stack_inuse, false);
|
|
}
|
|
|
|
static __always_inline void
|
|
__run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
|
|
struct pt_regs *regs)
|
|
{
|
|
void *tos = __this_cpu_read(hardirq_stack_ptr);
|
|
|
|
__this_cpu_write(hardirq_stack_inuse, true);
|
|
asm_call_sysvec_on_stack(tos, func, regs);
|
|
__this_cpu_write(hardirq_stack_inuse, false);
|
|
}
|
|
|
|
static __always_inline void
|
|
__run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
|
|
struct irq_desc *desc)
|
|
{
|
|
void *tos = __this_cpu_read(hardirq_stack_ptr);
|
|
|
|
__this_cpu_write(hardirq_stack_inuse, true);
|
|
asm_call_irq_on_stack(tos, func, desc);
|
|
__this_cpu_write(hardirq_stack_inuse, false);
|
|
}
|
|
|
|
#else /* CONFIG_X86_64 */
|
|
static inline bool irqstack_active(void) { return false; }
|
|
static inline void __run_on_irqstack(void (*func)(void)) { }
|
|
static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
|
|
struct pt_regs *regs) { }
|
|
static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
|
|
struct irq_desc *desc) { }
|
|
#endif /* !CONFIG_X86_64 */
|
|
|
|
static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs)
|
|
{
|
|
if (IS_ENABLED(CONFIG_X86_32))
|
|
return false;
|
|
if (!regs)
|
|
return !irqstack_active();
|
|
return !user_mode(regs) && !irqstack_active();
|
|
}
|
|
|
|
|
|
static __always_inline void run_on_irqstack_cond(void (*func)(void),
|
|
struct pt_regs *regs)
|
|
{
|
|
lockdep_assert_irqs_disabled();
|
|
|
|
if (irq_needs_irq_stack(regs))
|
|
__run_on_irqstack(func);
|
|
else
|
|
func();
|
|
}
|
|
|
|
static __always_inline void
|
|
run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs),
|
|
struct pt_regs *regs)
|
|
{
|
|
lockdep_assert_irqs_disabled();
|
|
|
|
if (irq_needs_irq_stack(regs))
|
|
__run_sysvec_on_irqstack(func, regs);
|
|
else
|
|
func(regs);
|
|
}
|
|
|
|
static __always_inline void
|
|
run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc,
|
|
struct pt_regs *regs)
|
|
{
|
|
lockdep_assert_irqs_disabled();
|
|
|
|
if (irq_needs_irq_stack(regs))
|
|
__run_irq_on_irqstack(func, desc);
|
|
else
|
|
func(desc);
|
|
}
|
|
|
|
#endif
|