printk: replace local_irq_save with local_lock for safe mode

Safe mode disables interrupts in order to minimize the window where
printk calls use deferred printing. Currently local_irq_save() is
used for this, however on PREEMPT_RT this can lead to large latencies
because safe mode is enabled for the duration of printing a record.

Use a local_lock instead of local_irq_save(). For !PREEMPT_RT it
has the same affect of disabling interrupts for that CPU. For
PREEMPT_RT it will disable preemption, which is enough to prevent
interruption from the irq threads.

Note that disabling preemption for PREEMPT_RT is also very bad
since it is still blocking RT tasks. The atomic/threaded (NOBKL)
consoles were developed such that safe mode is not needed. So it
is expected that a PREEMPT_RT machine does not run with any legacy
consoles registered.

Signed-off-by: John Ogness <john.ogness@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>

include/linux/printk.h

@@ -158,15 +158,17 @@ int _printk(const char *fmt, ...);
  */
 __printf(1, 2) __cold int _printk_deferred(const char *fmt, ...);
 
-extern void __printk_safe_enter(void);
+extern void __printk_safe_enter(unsigned long *flags);
-extern void __printk_safe_exit(void);
+extern void __printk_safe_exit(unsigned long *flags);
+extern void __printk_deferred_enter(void);
+extern void __printk_deferred_exit(void);
 /*
  * The printk_deferred_enter/exit macros are available only as a hack for
  * some code paths that need to defer all printk console printing. Interrupts
  * must be disabled for the deferred duration.
  */
-#define printk_deferred_enter __printk_safe_enter
+#define printk_deferred_enter() __printk_deferred_enter()
-#define printk_deferred_exit __printk_safe_exit
+#define printk_deferred_exit() __printk_deferred_exit()
 
 /*
  * Please don't use printk_ratelimit(), because it shares ratelimiting state

kernel/printk/internal.h

@@ -58,16 +58,24 @@ __printf(1, 0) int vprintk_deferred(const char *fmt, va_list args);
 
 bool printk_percpu_data_ready(void);
 
+/*
+ * The printk_safe_enter()/_exit() macros mark code blocks using locks that
+ * would lead to deadlock if an interrupting context were to call printk()
+ * while the interrupted context was within such code blocks.
+ *
+ * When a CPU is in such a code block, an interrupting context calling
+ * printk() will only log the new message to the lockless ringbuffer and
+ * then trigger console printing using irqwork.
+ */
+
 #define printk_safe_enter_irqsave(flags)	\
 	do {					\
-		local_irq_save(flags);		\
+		__printk_safe_enter(&flags);	\
-		__printk_safe_enter();		\
 	} while (0)
 
 #define printk_safe_exit_irqrestore(flags)	\
 	do {					\
-		__printk_safe_exit();		\
+		__printk_safe_exit(&flags);	\
-		local_irq_restore(flags);	\
 	} while (0)
 
 void defer_console_output(void);

kernel/printk/printk_safe.c

@@ -12,18 +12,41 @@
 
 #include "internal.h"
 
-static DEFINE_PER_CPU(int, printk_context);
+struct printk_context {
+	local_lock_t cpu;
+	int recursion;
+};
+
+static DEFINE_PER_CPU(struct printk_context, printk_context) = {
+	.cpu = INIT_LOCAL_LOCK(cpu),
+};
 
 /* Can be preempted by NMI. */
-void __printk_safe_enter(void)
+void __printk_safe_enter(unsigned long *flags)
 {
-	this_cpu_inc(printk_context);
+	WARN_ON_ONCE(in_nmi());
+	local_lock_irqsave(&printk_context.cpu, *flags);
+	this_cpu_inc(printk_context.recursion);
 }
 
 /* Can be preempted by NMI. */
-void __printk_safe_exit(void)
+void __printk_safe_exit(unsigned long *flags)
 {
-	this_cpu_dec(printk_context);
+	WARN_ON_ONCE(in_nmi());
+	this_cpu_dec(printk_context.recursion);
+	local_unlock_irqrestore(&printk_context.cpu, *flags);
+}
+
+void __printk_deferred_enter(void)
+{
+	WARN_ON_ONCE(!in_atomic());
+	this_cpu_inc(printk_context.recursion);
+}
+
+void __printk_deferred_exit(void)
+{
+	WARN_ON_ONCE(!in_atomic());
+	this_cpu_dec(printk_context.recursion);
 }
 
 asmlinkage int vprintk(const char *fmt, va_list args)
@@ -38,7 +61,7 @@ asmlinkage int vprintk(const char *fmt, va_list args)
 	 * Use the main logbuf even in NMI. But avoid calling console
 	 * drivers that might have their own locks.
 	 */
-	if (this_cpu_read(printk_context) || in_nmi())
+	if (this_cpu_read(printk_context.recursion) || in_nmi())
 		return vprintk_deferred(fmt, args);
 
 	/* No obstacles. */