[PATCH RFC v3 02/13] rcu-tasks-trace: Inline rcu_read_lock_trace() and annotate inside the reader

[Josef Bacik]

rcu_read_lock_trace() calls rcu_try_lock_acquire() before it has
entered the SRCU-fast reader, and rcu_read_unlock_trace() calls
srcu_lock_release() after it has left it.  rcu_read_lock() and
rcu_read_unlock() do it the other way around, annotating strictly
inside the critical section, and rcu_read_lock_tasks_trace() already
follows that order on the lock side.  Make the trace variants match.

Also make them, and the __srcu_read_lock_fast() and
__srcu_read_unlock_fast() they are built on, __always_inline like
rcu_read_lock() rather than leaving it to the compiler, which does
outline all four in KASAN/KCOV builds.

Besides consistency, this means the first thing a caller of
rcu_read_lock_trace() does is enter the reader and the last thing
rcu_read_unlock_trace() does is leave it, with no out-of-line call on
the outside.  A later patch relies on that for callers whose own text is
protected by the reader they are about to take.

Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@xxxxxxxxxxxxxx>
---
 include/linux/rcupdate_trace.h | 22 ++++++++++------------
 include/linux/srcutiny.h       |  4 ++--
 include/linux/srcutree.h       |  5 +++--
 3 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h
index 273c59a03251..4035054309d7 100644
--- a/include/linux/rcupdate_trace.h
+++ b/include/linux/rcupdate_trace.h
@@ -93,22 +93,20 @@ static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp)
  *
  * For more details, please see the documentation for rcu_read_lock().
  */
-static inline void rcu_read_lock_trace(void)
+static __always_inline void rcu_read_lock_trace(void)
 {
 	int n;
 	struct task_struct *t = current;
 
-	rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map);
 	n = READ_ONCE(t->trc_reader_nesting);
 	WRITE_ONCE(t->trc_reader_nesting, n + 1);
-	if (n) {
-		// In case we interrupted a Tasks Trace RCU reader.
-		return;
-	}
-	barrier();  // nesting before scp to protect against interrupt handler.
-	t->trc_reader_scp = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct);
-	if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB))
-		smp_mb(); // Placeholder for more selective ordering
+	if (!n) {
+		barrier();  // nesting before scp to protect against interrupt handler.
+		t->trc_reader_scp = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct);
+		if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB))
+			smp_mb(); // Placeholder for more selective ordering
+	} // Else we interrupted a Tasks Trace RCU reader.
+	rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map);
 }
 
 /**
@@ -120,12 +118,13 @@ static inline void rcu_read_lock_trace(void)
  *
  * For more details, please see the documentation for rcu_read_unlock().
  */
-static inline void rcu_read_unlock_trace(void)
+static __always_inline void rcu_read_unlock_trace(void)
 {
 	int n;
 	struct srcu_ctr __percpu *scp;
 	struct task_struct *t = current;
 
+	srcu_lock_release(&rcu_tasks_trace_srcu_struct.dep_map);
 	n = READ_ONCE(t->trc_reader_nesting) - 1;
 	if (n) {
 		WRITE_ONCE(t->trc_reader_nesting, n);
@@ -137,7 +136,6 @@ static inline void rcu_read_unlock_trace(void)
 			smp_mb(); // Placeholder for more selective ordering
 		__srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp);
 	}
-	srcu_lock_release(&rcu_tasks_trace_srcu_struct.dep_map);
 }
 
 /**
diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index fbcf13bc12d1..a43bae11c81c 100644
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h
@@ -101,13 +101,13 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss
 	return (struct srcu_ctr __percpu *)(intptr_t)idx;
 }
 
-static inline struct srcu_ctr __percpu *__srcu_read_lock_fast(struct srcu_struct *ssp)
+static __always_inline struct srcu_ctr __percpu *__srcu_read_lock_fast(struct srcu_struct *ssp)
 	__acquires_shared(ssp)
 {
 	return __srcu_ctr_to_ptr(ssp, __srcu_read_lock(ssp));
 }
 
-static inline void __srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
+static __always_inline void __srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
 	__releases_shared(ssp)
 {
 	__srcu_read_unlock(ssp, __srcu_ptr_to_ctr(ssp, scp));
diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 75e54e4f963f..fdb42ab50301 100644
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -286,7 +286,8 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss
  * on architectures that support NMIs but do not supply NMI-safe
  * implementations of this_cpu_inc().
  */
-static inline struct srcu_ctr __percpu notrace *__srcu_read_lock_fast(struct srcu_struct *ssp)
+static __always_inline struct srcu_ctr __percpu notrace *
+__srcu_read_lock_fast(struct srcu_struct *ssp)
 	__acquires_shared(ssp)
 {
 	struct srcu_ctr __percpu *scp = READ_ONCE(ssp->srcu_ctrp);
@@ -309,7 +310,7 @@ static inline struct srcu_ctr __percpu notrace *__srcu_read_lock_fast(struct src
  * Please see the __srcu_read_lock_fast() function's header comment for
  * information on implicit RCU readers and NMI safety.
  */
-static inline void notrace
+static __always_inline void notrace
 __srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
 	__releases_shared(ssp)
 {

-- 
2.55.0