[PATCH 1/3] sched/tick: Decouple sched_tick() from HZ

[Qais Yousef]

Add new sched_feat(PTICK), Pseudo Tick, which runs at a more regular 1ms
interval when a task is running ensuring more timely update of sched
stats.

As discussed in previous attempt to change default HZ to 1000 [1], it is
very consequential on scheduler decisions. By decoupling sched_tick()
from HZ (timer tick), scheduler will no longer be limited by other TICK
implications and can be set to a constant value, 1000.

We could make it configurable, if folks think it can be useful, but this
seems to defeat the purpose of introducing it. We want fast regular
ticks. It can be turned off via sched_features for folks who don't want
the potential additional overhead. Defaults off for now.

PTICK will only be active when there are tasks running on the CPU. Given
HRTICK is now default on, it'd be more consequential for solo long
running tasks. When HRTICK off, it'd help with more precise time slices
when HZ is low.

When no tasks are running, sched_tick() will still be triggered from
regular TICK.

The actual interval is actually 1025us to ensure a PELT period (1024us)
has passed.

Since load balancer is tied to jiffies, we need additional work to
decouple it and help with faster reaction. Circulating patches to
introduce push load balancer could make this conversion unnecessary.

Other users of jiffies might find it useful to use the new ptick. But
any jiffies related conversion will be left out for now.

[1] https://lore.kernel.org/lkml/20250226000810.459547-1-qyousef@xxxxxxxxxxx/

Signed-off-by: Qais Yousef <qyousef@xxxxxxxxxxx>
---
 kernel/sched/core.c     | 78 +++++++++++++++++++++++++++++++++++++----
 kernel/sched/features.h |  7 ++++
 kernel/sched/sched.h    |  7 ++++
 3 files changed, 86 insertions(+), 6 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index b905805bbcbe..b47ac98ff42b 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -890,7 +890,7 @@ static void __used hrtick_clear(struct rq *rq)
 }
 
 /*
- * High-resolution timer tick.
+ * High-resolution timer tick. Triggers at the end of next_task slice.
  * Runs from hardirq context with interrupts disabled.
  */
 static enum hrtimer_restart hrtick(struct hrtimer *timer)
@@ -1011,11 +1011,63 @@ static void hrtick_rq_init(struct rq *rq)
 	hrtimer_setup(&rq->hrtick_timer, hrtick, CLOCK_MONOTONIC,
 		      HRTIMER_MODE_REL_HARD | HRTIMER_MODE_LAZY_REARM);
 }
+
+static void ptick_clear(struct rq *rq)
+{
+	if (hrtimer_active(&rq->ptick_timer))
+		hrtimer_cancel(&rq->ptick_timer);
+}
+
+static void ptick_start(struct rq *rq)
+{
+	hrtimer_start(&rq->ptick_timer, ns_to_ktime(PTICK_NSEC),
+		      HRTIMER_MODE_ABS_PINNED_HARD);
+}
+
+static void ptick_ctx_switch(struct rq *rq,
+			     struct task_struct *prev, struct task_struct *next)
+{
+	if (is_idle_task(next))
+		ptick_clear(rq);
+
+	if (sched_feat(PTICK) && is_idle_task(prev))
+		ptick_start(rq);
+}
+
+void __sched_tick(void);
+
+/*
+ * High-resolution timer tick. Triggers regularly every PTICK_NSEC.
+ * Runs from hardirq context with interrupts disabled.
+ */
+static enum hrtimer_restart ptick(struct hrtimer *timer)
+{
+	struct rq *rq = container_of(timer, struct rq, ptick_timer);
+
+	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
+
+	__sched_tick();
+
+	hrtimer_forward_now(timer, ns_to_ktime(PTICK_NSEC));
+
+	return HRTIMER_RESTART;
+}
+
+static void ptick_rq_init(struct rq *rq)
+{
+	hrtimer_setup(&rq->ptick_timer, ptick, CLOCK_MONOTONIC,
+		      HRTIMER_MODE_REL_HARD | HRTIMER_MODE_LAZY_REARM);
+}
 #else /* !CONFIG_SCHED_HRTICK: */
 static inline void hrtick_clear(struct rq *rq) { }
 static inline void hrtick_rq_init(struct rq *rq) { }
 static inline void hrtick_schedule_enter(struct rq *rq) { }
 static inline void hrtick_schedule_exit(struct rq *rq) { }
+static inline void ptick_clear(struct rq *rq) { }
+static inline void ptick_start(struct rq *rq) { }
+static inline void ptick_ctx_switch(struct rq *rq,
+				    struct task_struct *prev, struct task_struct *next) { }
+static inline void ptick_rq_init(struct rq *rq) { }
 #endif /* !CONFIG_SCHED_HRTICK */
 
 /*
@@ -5246,6 +5298,7 @@ static struct rq *finish_task_switch(struct task_struct *prev)
 	prev_state = READ_ONCE(prev->__state);
 	vtime_task_switch(prev);
 	perf_event_task_sched_in(prev, current);
+	ptick_ctx_switch(rq, prev, current);
 	finish_task(prev);
 	tick_nohz_task_switch();
 	finish_lock_switch(rq);
@@ -5637,11 +5690,7 @@ static int __init setup_resched_latency_warn_ms(char *str)
 }
 __setup("resched_latency_warn_ms=", setup_resched_latency_warn_ms);
 
-/*
- * This function gets called by the timer code, with HZ frequency.
- * We call it with interrupts disabled.
- */
-void sched_tick(void)
+void __sched_tick(void)
 {
 	int cpu = smp_processor_id();
 	struct rq *rq = cpu_rq(cpu);
@@ -5691,6 +5740,22 @@ void sched_tick(void)
 	}
 }
 
+/*
+ * This function gets called by the timer code, with HZ frequency or
+ * PTICK_NSEC. We call it with interrupts disabled.
+ */
+void sched_tick(void)
+{
+	int cpu = smp_processor_id();
+	struct rq *rq = cpu_rq(cpu);
+
+	/* Skip HZ calls unless there are no tasks running when PTICK is enabled */
+	if (sched_feat(PTICK) && rq->nr_running)
+		return;
+
+	__sched_tick();
+}
+
 #ifdef CONFIG_NO_HZ_FULL
 
 struct tick_work {
@@ -9017,6 +9082,7 @@ void __init sched_init(void)
 		rcuwait_init(&rq->hotplug_wait);
 #endif
 		hrtick_rq_init(rq);
+		ptick_rq_init(rq);
 		atomic_set(&rq->nr_iowait, 0);
 		fair_server_init(rq);
 #ifdef CONFIG_SCHED_CLASS_EXT
diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index 84c4fe3abd74..dcce2558ac21 100644
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -73,6 +73,13 @@ SCHED_FEAT(HRTICK, false)
 SCHED_FEAT(HRTICK_DL, false)
 #endif
 
+/*
+ * Pseudo Tick. Triggers when tasks are running on CPU. sched_tick() will run
+ * off of it when turned on. Will use regular TICK when no tasks are running or
+ * turned off.
+ */
+SCHED_FEAT(PTICK, false)
+
 /*
  * Decrement CPU capacity based on time not spent running tasks
  */
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 9f63b15d309d..78ea50905331 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1297,6 +1297,8 @@ struct rq {
 	ktime_t			hrtick_time;
 	ktime_t			hrtick_delay;
 	unsigned int		hrtick_sched;
+
+	struct hrtimer		ptick_timer;
 #endif
 
 #ifdef CONFIG_SCHEDSTATS
@@ -3070,6 +3072,11 @@ extern unsigned int sysctl_numa_balancing_hot_threshold;
 
 #ifdef CONFIG_SCHED_HRTICK
 
+/* PELT period is 1024us, add 1 to ensure one PELT period has passed */
+#define PTICK_USEC		(sched_feat(PTICK) ? (long)(USEC_PER_MSEC * 1.025) : TICK_USEC)
+#define PTICK_NSEC		(sched_feat(PTICK) ? (long)(NSEC_PER_MSEC * 1.025) : TICK_NSEC)
+#define PTICK_HZ		(sched_feat(PTICK) ? 1000 : HZ)
+
 /*
  * Use hrtick when:
  *  - enabled by features
-- 
2.34.1