mirror of
https://github.com/raspberrypi/linux.git
synced 2025-12-19 16:24:21 +00:00
d021985504
Background:
===========
Currently kernel-mode FPU is not always usable in softirq context on
x86, since softirqs can nest inside a kernel-mode FPU section in task
context, and nested use of kernel-mode FPU is not supported.
Therefore, x86 SIMD-optimized code that can be called in softirq context
has to sometimes fall back to non-SIMD code. There are two options for
the fallback, both of which are pretty terrible:
(a) Use a scalar fallback. This can be 10-100x slower than vectorized
code because it cannot use specialized instructions like AES, SHA,
or carryless multiplication.
(b) Execute the request asynchronously using a kworker. In other
words, use the "crypto SIMD helper" in crypto/simd.c.
Currently most of the x86 en/decryption code (skcipher and aead
algorithms) uses option (b), since this avoids the slow scalar fallback
and it is easier to wire up. But option (b) is still really bad for its
own reasons:
- Punting the request to a kworker is bad for performance too.
- It forces the algorithm to be marked as asynchronous
(CRYPTO_ALG_ASYNC), preventing it from being used by crypto API
users who request a synchronous algorithm. That's another huge
performance problem, which is especially unfortunate for users who
don't even do en/decryption in softirq context.
- It makes all en/decryption operations take a detour through
crypto/simd.c. That involves additional checks and an additional
indirect call, which slow down en/decryption for *everyone*.
Fortunately, the skcipher and aead APIs are only usable in task and
softirq context in the first place. Thus, if kernel-mode FPU were to be
reliably usable in softirq context, no fallback would be needed.
Indeed, other architectures such as arm, arm64, and riscv have already
done this.
Changes implemented:
====================
Therefore, this patch updates x86 accordingly to reliably support
kernel-mode FPU in softirqs.
This is done by just disabling softirq processing in kernel-mode FPU
sections (when hardirqs are not already disabled), as that prevents the
nesting that was problematic.
This will delay some softirqs slightly, but only ones that would have
otherwise been nested inside a task context kernel-mode FPU section.
Any such softirqs would have taken the slow fallback path before if they
tried to do any en/decryption. Now these softirqs will just run at the
end of the task context kernel-mode FPU section (since local_bh_enable()
runs pending softirqs) and will no longer take the slow fallback path.
Alternatives considered:
========================
- Make kernel-mode FPU sections fully preemptible. This would require
growing task_struct by another struct fpstate which is more than 2K.
- Make softirqs save/restore the kernel-mode FPU state to a per-CPU
struct fpstate when nested use is detected. Somewhat interesting, but
seems unnecessary when a simpler solution exists.
Performance results:
====================
I did some benchmarks with AES-XTS encryption of 16-byte messages (which is
unrealistically small, but this makes it easier to see the overhead of
kernel-mode FPU...). The baseline was 384 MB/s. Removing the use of
crypto/simd.c, which this work makes possible, increases it to 487 MB/s,
a +27% improvement in throughput.
CPU was AMD Ryzen 9 9950X (Zen 5). No debugging options were enabled.
[ mingo: Prettified the changelog and added performance results. ]
Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Uros Bizjak <ubizjak@gmail.com>
Link: https://lore.kernel.org/r/20250304204954.3901-1-ebiggers@kernel.org
180 lines
5.6 KiB
C
180 lines
5.6 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* Copyright (C) 1994 Linus Torvalds
|
|
*
|
|
* Pentium III FXSR, SSE support
|
|
* General FPU state handling cleanups
|
|
* Gareth Hughes <gareth@valinux.com>, May 2000
|
|
* x86-64 work by Andi Kleen 2002
|
|
*/
|
|
|
|
#ifndef _ASM_X86_FPU_API_H
|
|
#define _ASM_X86_FPU_API_H
|
|
#include <linux/bottom_half.h>
|
|
|
|
#include <asm/fpu/types.h>
|
|
|
|
/*
|
|
* Use kernel_fpu_begin/end() if you intend to use FPU in kernel context. It
|
|
* disables preemption and softirq processing, so be careful if you intend to
|
|
* use it for long periods of time. Kernel-mode FPU cannot be used in all
|
|
* contexts -- see irq_fpu_usable() for details.
|
|
*/
|
|
|
|
/* Kernel FPU states to initialize in kernel_fpu_begin_mask() */
|
|
#define KFPU_387 _BITUL(0) /* 387 state will be initialized */
|
|
#define KFPU_MXCSR _BITUL(1) /* MXCSR will be initialized */
|
|
|
|
extern void kernel_fpu_begin_mask(unsigned int kfpu_mask);
|
|
extern void kernel_fpu_end(void);
|
|
extern bool irq_fpu_usable(void);
|
|
extern void fpregs_mark_activate(void);
|
|
|
|
/* Code that is unaware of kernel_fpu_begin_mask() can use this */
|
|
static inline void kernel_fpu_begin(void)
|
|
{
|
|
#ifdef CONFIG_X86_64
|
|
/*
|
|
* Any 64-bit code that uses 387 instructions must explicitly request
|
|
* KFPU_387.
|
|
*/
|
|
kernel_fpu_begin_mask(KFPU_MXCSR);
|
|
#else
|
|
/*
|
|
* 32-bit kernel code may use 387 operations as well as SSE2, etc,
|
|
* as long as it checks that the CPU has the required capability.
|
|
*/
|
|
kernel_fpu_begin_mask(KFPU_387 | KFPU_MXCSR);
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Use fpregs_lock() while editing CPU's FPU registers or fpu->fpstate, or while
|
|
* using the FPU in kernel mode. A context switch will (and softirq might) save
|
|
* CPU's FPU registers to fpu->fpstate.regs and set TIF_NEED_FPU_LOAD leaving
|
|
* CPU's FPU registers in a random state.
|
|
*
|
|
* local_bh_disable() protects against both preemption and soft interrupts
|
|
* on !RT kernels.
|
|
*
|
|
* On RT kernels local_bh_disable() is not sufficient because it only
|
|
* serializes soft interrupt related sections via a local lock, but stays
|
|
* preemptible. Disabling preemption is the right choice here as bottom
|
|
* half processing is always in thread context on RT kernels so it
|
|
* implicitly prevents bottom half processing as well.
|
|
*/
|
|
static inline void fpregs_lock(void)
|
|
{
|
|
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
|
|
local_bh_disable();
|
|
else
|
|
preempt_disable();
|
|
}
|
|
|
|
static inline void fpregs_unlock(void)
|
|
{
|
|
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
|
|
local_bh_enable();
|
|
else
|
|
preempt_enable();
|
|
}
|
|
|
|
/*
|
|
* FPU state gets lazily restored before returning to userspace. So when in the
|
|
* kernel, the valid FPU state may be kept in the buffer. This function will force
|
|
* restore all the fpu state to the registers early if needed, and lock them from
|
|
* being automatically saved/restored. Then FPU state can be modified safely in the
|
|
* registers, before unlocking with fpregs_unlock().
|
|
*/
|
|
void fpregs_lock_and_load(void);
|
|
|
|
#ifdef CONFIG_X86_DEBUG_FPU
|
|
extern void fpregs_assert_state_consistent(void);
|
|
#else
|
|
static inline void fpregs_assert_state_consistent(void) { }
|
|
#endif
|
|
|
|
/*
|
|
* Load the task FPU state before returning to userspace.
|
|
*/
|
|
extern void switch_fpu_return(void);
|
|
|
|
/*
|
|
* Query the presence of one or more xfeatures. Works on any legacy CPU as well.
|
|
*
|
|
* If 'feature_name' is set then put a human-readable description of
|
|
* the feature there as well - this can be used to print error (or success)
|
|
* messages.
|
|
*/
|
|
extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name);
|
|
|
|
/* Trap handling */
|
|
extern int fpu__exception_code(struct fpu *fpu, int trap_nr);
|
|
extern void fpu_sync_fpstate(struct fpu *fpu);
|
|
extern void fpu_reset_from_exception_fixup(void);
|
|
|
|
/* Boot, hotplug and resume */
|
|
extern void fpu__init_cpu(void);
|
|
extern void fpu__init_system(void);
|
|
extern void fpu__init_check_bugs(void);
|
|
extern void fpu__resume_cpu(void);
|
|
|
|
#ifdef CONFIG_MATH_EMULATION
|
|
extern void fpstate_init_soft(struct swregs_state *soft);
|
|
#else
|
|
static inline void fpstate_init_soft(struct swregs_state *soft) {}
|
|
#endif
|
|
|
|
/* State tracking */
|
|
DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
|
|
|
|
/* Process cleanup */
|
|
#ifdef CONFIG_X86_64
|
|
extern void fpstate_free(struct fpu *fpu);
|
|
#else
|
|
static inline void fpstate_free(struct fpu *fpu) { }
|
|
#endif
|
|
|
|
/* fpstate-related functions which are exported to KVM */
|
|
extern void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature);
|
|
|
|
extern u64 xstate_get_guest_group_perm(void);
|
|
|
|
extern void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr);
|
|
|
|
|
|
/* KVM specific functions */
|
|
extern bool fpu_alloc_guest_fpstate(struct fpu_guest *gfpu);
|
|
extern void fpu_free_guest_fpstate(struct fpu_guest *gfpu);
|
|
extern int fpu_swap_kvm_fpstate(struct fpu_guest *gfpu, bool enter_guest);
|
|
extern int fpu_enable_guest_xfd_features(struct fpu_guest *guest_fpu, u64 xfeatures);
|
|
|
|
#ifdef CONFIG_X86_64
|
|
extern void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd);
|
|
extern void fpu_sync_guest_vmexit_xfd_state(void);
|
|
#else
|
|
static inline void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd) { }
|
|
static inline void fpu_sync_guest_vmexit_xfd_state(void) { }
|
|
#endif
|
|
|
|
extern void fpu_copy_guest_fpstate_to_uabi(struct fpu_guest *gfpu, void *buf,
|
|
unsigned int size, u64 xfeatures, u32 pkru);
|
|
extern int fpu_copy_uabi_to_guest_fpstate(struct fpu_guest *gfpu, const void *buf, u64 xcr0, u32 *vpkru);
|
|
|
|
static inline void fpstate_set_confidential(struct fpu_guest *gfpu)
|
|
{
|
|
gfpu->fpstate->is_confidential = true;
|
|
}
|
|
|
|
static inline bool fpstate_is_confidential(struct fpu_guest *gfpu)
|
|
{
|
|
return gfpu->fpstate->is_confidential;
|
|
}
|
|
|
|
/* prctl */
|
|
extern long fpu_xstate_prctl(int option, unsigned long arg2);
|
|
|
|
extern void fpu_idle_fpregs(void);
|
|
|
|
#endif /* _ASM_X86_FPU_API_H */
|