From 11e11d0a7d68a3db04edebbaae19d7c01cea0ca1 Mon Sep 17 00:00:00 2001 From: Sebastian Siewior Date: Tue, 14 Mar 2023 16:51:44 +0100 Subject: [PATCH] printk: replace local_irq_save with local_lock for safe mode Safe mode disables interrupts in order to minimize the window where printk calls use deferred printing. Currently local_irq_save() is used for this, however on PREEMPT_RT this can lead to large latencies because safe mode is enabled for the duration of printing a record. Use a local_lock instead of local_irq_save(). For !PREEMPT_RT it has the same affect of disabling interrupts for that CPU. For PREEMPT_RT it will disable preemption, which is enough to prevent interruption from the irq threads. Note that disabling preemption for PREEMPT_RT is also very bad since it is still blocking RT tasks. The atomic/threaded (NOBKL) consoles were developed such that safe mode is not needed. So it is expected that a PREEMPT_RT machine does not run with any legacy consoles registered. Signed-off-by: John Ogness Signed-off-by: Sebastian Andrzej Siewior --- include/linux/printk.h | 10 ++++++---- kernel/printk/internal.h | 16 ++++++++++++---- kernel/printk/printk_safe.c | 35 +++++++++++++++++++++++++++++------ 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/include/linux/printk.h b/include/linux/printk.h index d2aafc79b611..b55662624ff8 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -158,15 +158,17 @@ int _printk(const char *fmt, ...); */ __printf(1, 2) __cold int _printk_deferred(const char *fmt, ...); -extern void __printk_safe_enter(void); -extern void __printk_safe_exit(void); +extern void __printk_safe_enter(unsigned long *flags); +extern void __printk_safe_exit(unsigned long *flags); +extern void __printk_deferred_enter(void); +extern void __printk_deferred_exit(void); /* * The printk_deferred_enter/exit macros are available only as a hack for * some code paths that need to defer all printk console printing. Interrupts * must be disabled for the deferred duration. */ -#define printk_deferred_enter __printk_safe_enter -#define printk_deferred_exit __printk_safe_exit +#define printk_deferred_enter() __printk_deferred_enter() +#define printk_deferred_exit() __printk_deferred_exit() /* * Please don't use printk_ratelimit(), because it shares ratelimiting state diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 6b21fe0a38f5..2d47a69849e2 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -58,16 +58,24 @@ __printf(1, 0) int vprintk_deferred(const char *fmt, va_list args); bool printk_percpu_data_ready(void); +/* + * The printk_safe_enter()/_exit() macros mark code blocks using locks that + * would lead to deadlock if an interrupting context were to call printk() + * while the interrupted context was within such code blocks. + * + * When a CPU is in such a code block, an interrupting context calling + * printk() will only log the new message to the lockless ringbuffer and + * then trigger console printing using irqwork. + */ + #define printk_safe_enter_irqsave(flags) \ do { \ - local_irq_save(flags); \ - __printk_safe_enter(); \ + __printk_safe_enter(&flags); \ } while (0) #define printk_safe_exit_irqrestore(flags) \ do { \ - __printk_safe_exit(); \ - local_irq_restore(flags); \ + __printk_safe_exit(&flags); \ } while (0) void defer_console_output(void); diff --git a/kernel/printk/printk_safe.c b/kernel/printk/printk_safe.c index 6d10927a07d8..5c1470bd60bc 100644 --- a/kernel/printk/printk_safe.c +++ b/kernel/printk/printk_safe.c @@ -12,18 +12,41 @@ #include "internal.h" -static DEFINE_PER_CPU(int, printk_context); +struct printk_context { + local_lock_t cpu; + int recursion; +}; + +static DEFINE_PER_CPU(struct printk_context, printk_context) = { + .cpu = INIT_LOCAL_LOCK(cpu), +}; /* Can be preempted by NMI. */ -void __printk_safe_enter(void) +void __printk_safe_enter(unsigned long *flags) { - this_cpu_inc(printk_context); + WARN_ON_ONCE(in_nmi()); + local_lock_irqsave(&printk_context.cpu, *flags); + this_cpu_inc(printk_context.recursion); } /* Can be preempted by NMI. */ -void __printk_safe_exit(void) +void __printk_safe_exit(unsigned long *flags) { - this_cpu_dec(printk_context); + WARN_ON_ONCE(in_nmi()); + this_cpu_dec(printk_context.recursion); + local_unlock_irqrestore(&printk_context.cpu, *flags); +} + +void __printk_deferred_enter(void) +{ + WARN_ON_ONCE(!in_atomic()); + this_cpu_inc(printk_context.recursion); +} + +void __printk_deferred_exit(void) +{ + WARN_ON_ONCE(!in_atomic()); + this_cpu_dec(printk_context.recursion); } asmlinkage int vprintk(const char *fmt, va_list args) @@ -38,7 +61,7 @@ asmlinkage int vprintk(const char *fmt, va_list args) * Use the main logbuf even in NMI. But avoid calling console * drivers that might have their own locks. */ - if (this_cpu_read(printk_context) || in_nmi()) + if (this_cpu_read(printk_context.recursion) || in_nmi()) return vprintk_deferred(fmt, args); /* No obstacles. */