[PATCH 6.18.y 1/6] irq: Add refcounted local interrupt disabling

[Alice Ryhl]

Rust's SpinLockIrq abstraction requires refcounted local interrupt
disabling so that nested critical sections can be released in any order.
Upstream commit e901c1510e24 ("irq,spin_lock: Add counted interrupt
disabling/enabling") implements this by carving HARDIRQ_DISABLE_BITS out
of preempt_count, which requires invasive architecture-specific changes
across x86, arm64, and s390. Thus, for 6.18, implement the same API
using a dedicated per-CPU nesting count and saved irqflags word instead.

Signed-off-by: Alice Ryhl <aliceryhl@xxxxxxxxxx>
---
 include/linux/interrupt_rc.h | 26 ++++++++++++++++++
 include/linux/spinlock.h     | 61 ++++++++++++++++++++++++++++++++++++++++++
 kernel/softirq.c             | 63 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 150 insertions(+)

diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
new file mode 100644
index 000000000000..64564bfd715a
--- /dev/null
+++ b/include/linux/interrupt_rc.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __LINUX_INTERRUPT_RC_H
+#define __LINUX_INTERRUPT_RC_H
+
+/*
+ * Refcounted local processor interrupt management.
+ *
+ * local_interrupt_disable() disables interrupts on the local CPU and may be
+ * nested; interrupts are only restored once the outermost
+ * local_interrupt_enable() runs.
+ *
+ * Unlike local_irq_save()/local_irq_restore(), the saved interrupt state is
+ * held in per-CPU state rather than by the caller, so the regions do not have
+ * to be released in acquisition order. That property is what allows Rust's
+ * SpinLockIrq guards - which are ordinary values and may be dropped in any
+ * order - to be built on top of this.
+ *
+ * Since the implementation depends on local_irq_save()/local_irq_restore(),
+ * this lives in its own header to make it easier to include without hitting
+ * circular header dependencies.
+ */
+
+void local_interrupt_disable(void);
+void local_interrupt_enable(void);
+
+#endif /* !__LINUX_INTERRUPT_RC_H */
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index d3561c4a080e..bb067c652fbd 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -57,6 +57,7 @@
 #include <linux/linkage.h>
 #include <linux/compiler.h>
 #include <linux/irqflags.h>
+#include <linux/interrupt_rc.h>
 #include <linux/thread_info.h>
 #include <linux/stringify.h>
 #include <linux/bottom_half.h>
@@ -455,6 +456,66 @@ static __always_inline int spin_is_contended(spinlock_t *lock)
 # include <linux/spinlock_rt.h>
 #endif /* CONFIG_PREEMPT_RT */
 
+/*
+ * spin_lock_irq_disable() and friends behave like spin_lock_irq(), except that
+ * the interrupt disabling is refcounted (see <linux/interrupt_rc.h>).
+ *
+ * Two differences from spin_lock_irq() follow from that:
+ *
+ *  - Nesting works: an inner critical section does not re-enable interrupts on
+ *    exit while an outer one is still held.
+ *  - If interrupts were already disabled on entry to the outermost section,
+ *    they stay disabled on exit, as with spin_lock_irqsave().
+ *
+ * Unlike spin_lock_irqsave() the saved state lives in per-CPU state rather
+ * than in a caller-provided variable, so sections need not be released in
+ * acquisition order. These exist for Rust's SpinLockIrq, whose guards are
+ * ordinary values and may be dropped in any order.
+ */
+#ifndef CONFIG_PREEMPT_RT
+static __always_inline void spin_lock_irq_disable(spinlock_t *lock)
+{
+	local_interrupt_disable();
+	spin_lock(lock);
+}
+
+static __always_inline void spin_unlock_irq_enable(spinlock_t *lock)
+{
+	spin_unlock(lock);
+	local_interrupt_enable();
+}
+
+static __always_inline int spin_trylock_irq_disable(spinlock_t *lock)
+{
+	local_interrupt_disable();
+
+	if (spin_trylock(lock))
+		return 1;
+
+	local_interrupt_enable();
+	return 0;
+}
+#else /* CONFIG_PREEMPT_RT */
+/*
+ * On PREEMPT_RT spinlock_t is a sleeping lock and spin_lock_irq() does not
+ * disable interrupts at all; mirror that behaviour here.
+ */
+static __always_inline void spin_lock_irq_disable(spinlock_t *lock)
+{
+	spin_lock(lock);
+}
+
+static __always_inline void spin_unlock_irq_enable(spinlock_t *lock)
+{
+	spin_unlock(lock);
+}
+
+static __always_inline int spin_trylock_irq_disable(spinlock_t *lock)
+{
+	return spin_trylock(lock);
+}
+#endif /* CONFIG_PREEMPT_RT */
+
 /*
  * Does a critical section need to be broken due to another
  * task waiting?: (technically does not depend on CONFIG_PREEMPTION,
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 77198911b8dd..3d17cf5ff790 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -12,6 +12,7 @@
 #include <linux/export.h>
 #include <linux/kernel_stat.h>
 #include <linux/interrupt.h>
+#include <linux/interrupt_rc.h>
 #include <linux/init.h>
 #include <linux/local_lock.h>
 #include <linux/mm.h>
@@ -88,6 +89,68 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
 EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
 #endif
 
+/*
+ * Refcounted local interrupt disabling; see <linux/interrupt_rc.h>.
+ *
+ * @local_interrupt_disable_count is the nesting depth on this CPU and
+ * @local_interrupt_disable_flags holds the irqflags captured by the outermost
+ * local_interrupt_disable().
+ *
+ * Both are only ever accessed by the owning CPU with interrupts disabled, so
+ * no atomics are needed: local_interrupt_disable() disables interrupts before
+ * touching the count, and a non-zero count implies interrupts are disabled.
+ */
+static DEFINE_PER_CPU(unsigned int, local_interrupt_disable_count);
+static DEFINE_PER_CPU(unsigned long, local_interrupt_disable_flags);
+
+void local_interrupt_disable(void)
+{
+	unsigned int count;
+	unsigned long flags;
+
+	WARN_ON_ONCE(in_nmi());
+
+	/*
+	 * Disable interrupts before touching the count, so that an interrupt
+	 * handler on this CPU cannot observe or modify a half-updated state.
+	 */
+	local_irq_save(flags);
+
+	count = raw_cpu_read(local_interrupt_disable_count);
+	if (count == 0) {
+		preempt_disable();
+		raw_cpu_write(local_interrupt_disable_flags, flags);
+	}
+
+	raw_cpu_write(local_interrupt_disable_count, count + 1);
+}
+EXPORT_SYMBOL_GPL(local_interrupt_disable);
+
+void local_interrupt_enable(void)
+{
+	unsigned int count = raw_cpu_read(local_interrupt_disable_count);
+
+	if (WARN_ON_ONCE(count == 0))
+		return;
+
+	count--;
+	raw_cpu_write(local_interrupt_disable_count, count);
+
+	if (count == 0) {
+		unsigned long flags = raw_cpu_read(local_interrupt_disable_flags);
+
+		/*
+		 * Restore rather than unconditionally enable: the outermost
+		 * local_interrupt_disable() may itself have run with
+		 * interrupts already disabled, in which case they must stay
+		 * disabled here.
+		 */
+		local_irq_restore(flags);
+		preempt_enable();
+	}
+}
+EXPORT_SYMBOL_GPL(local_interrupt_enable);
+
 /*
  * SOFTIRQ_OFFSET usage:
  *

-- 
2.56.0.385.gd3acb90ef8-goog