[RFC PATCH 37/46] KVM: VMX: Add Caretaker VMX assembly guest entry/exit routine and helpers
From: Pasha Tatashin
Date: Sun Sep 20 2026 - 16:04:46 EST
Add arch/x86/kvm/vmx/caretaker_vmenter.S and caretaker.h for standalone
Intel VMX VMLAUNCH/VMRESUME world switch on preserved physical cores.
Signed-off-by: Pasha Tatashin <pasha.tatashin@xxxxxxxxxx>
---
arch/x86/kvm/msrs.c | 2 +
arch/x86/kvm/vmx/caretaker.h | 78 +++++++++++++++++++
arch/x86/kvm/vmx/caretaker_vmenter.S | 112 +++++++++++++++++++++++++++
3 files changed, 192 insertions(+)
create mode 100644 arch/x86/kvm/vmx/caretaker.h
create mode 100644 arch/x86/kvm/vmx/caretaker_vmenter.S
diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c
index ae849e0b7d14..536c8d1959dc 100644
--- a/arch/x86/kvm/msrs.c
+++ b/arch/x86/kvm/msrs.c
@@ -882,11 +882,13 @@ int kvm_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data)
{
return __kvm_set_msr(vcpu, index, data, true);
}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_msr_write);
int kvm_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data)
{
return __kvm_get_msr(vcpu, index, data, true);
}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_msr_read);
int __kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data)
{
diff --git a/arch/x86/kvm/vmx/caretaker.h b/arch/x86/kvm/vmx/caretaker.h
new file mode 100644
index 000000000000..d4272e202e73
--- /dev/null
+++ b/arch/x86/kvm/vmx/caretaker.h
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2026, Google LLC.
+ * Pasha Tatashin <pasha.tatashin@xxxxxxxxxx>
+ */
+#ifndef __VMX_CARETAKER_H
+#define __VMX_CARETAKER_H
+
+#define VMCS_VM_INSTRUCTION_ERROR 0x4400
+
+#ifndef __ASSEMBLY__
+#include <linux/types.h>
+#include <linux/kvm_host.h>
+#include <linux/oncore.h>
+#include <asm/vmx.h>
+#endif
+
+#include "../caretaker.h"
+
+#ifndef __ASSEMBLY__
+#include <asm/desc.h>
+#include <linux/processor.h>
+
+/* Length of instructions */
+#define PAUSE_INSN_LEN 2
+#define VMCALL_INSN_LEN 3
+
+/* Exit qualification bitfields for I/O instructions (Intel SDM Vol 3C) */
+#define VMX_IO_SIZE_MASK 0x7
+#define VMX_IO_DIRECTION_BIT BIT(3)
+#define VMX_IO_PORT_SHIFT 16
+
+/* Default VMX preemption timer shift (counts down every 2^5 TSC ticks) */
+#define VMX_PREEMPTION_TIMER_SHIFT 5
+
+struct caretaker_vmx_page {
+ struct caretaker_x86_page common;
+ /* Guest syscall state not automatically switched by VMCS */
+ u64 star;
+ u64 lstar;
+ u64 fmask;
+ u64 vmxon_pa;
+ u32 timer_shift;
+ u32 ple_supported;
+ u8 vmxon_area[PAGE_SIZE] __aligned(PAGE_SIZE);
+} __aligned(PAGE_SIZE);
+
+static inline unsigned long vmx_vmread(unsigned long field)
+{
+ unsigned long val;
+
+ asm volatile("vmread %1, %0" : "=r" (val) : "r" (field) : "cc");
+ return val;
+}
+
+static inline void vmx_vmwrite(unsigned long field, unsigned long val)
+{
+ asm volatile("vmwrite %1, %0" : : "r" (field), "r" (val) : "cc");
+}
+
+#ifdef CONFIG_KVM_CARETAKER
+int vmx_caretaker_enter(void *page);
+void vmx_caretaker_exit_handler(void);
+void vmx_caretaker_register(void);
+void vmx_caretaker_unregister(void);
+void vmx_caretaker_decode_exit(void *page,
+ struct kvm_caretaker_exit *exit);
+void vmx_caretaker_init_host_vmcs(struct caretaker_vmx_page *cvp);
+void vmx_caretaker_init(struct kvm_vcpu *vcpu);
+#else
+static inline void vmx_caretaker_register(void) {}
+static inline void vmx_caretaker_unregister(void) {}
+static inline void vmx_caretaker_init(struct kvm_vcpu *vcpu) {}
+#endif
+
+#endif /* !__ASSEMBLY__ */
+
+#endif /* __VMX_CARETAKER_H */
diff --git a/arch/x86/kvm/vmx/caretaker_vmenter.S b/arch/x86/kvm/vmx/caretaker_vmenter.S
new file mode 100644
index 000000000000..ff135cba854d
--- /dev/null
+++ b/arch/x86/kvm/vmx/caretaker_vmenter.S
@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026, Google LLC.
+ * Pasha Tatashin <pasha.tatashin@xxxxxxxxxx>
+ */
+#include <linux/linkage.h>
+#include <linux/cfi_types.h>
+#include <linux/objtool.h>
+#include <asm/asm.h>
+#include <asm/bitsperlong.h>
+#include <asm/kvm_vcpu_regs.h>
+#include <asm/nospec-branch.h>
+#include <asm/msr-index.h>
+#include <asm/page_types.h>
+#include "caretaker.h"
+#include "../caretaker_asm.h"
+
+.section .text.cpu_preserved, "ax"
+
+/**
+ * vmx_caretaker_enter - Low-level transition to VMX guest mode and back
+ * @cvp: Pointer to struct caretaker_vmx_page (%rdi)
+ */
+SYM_TYPED_FUNC_START(vmx_caretaker_enter)
+ ENDBR
+ CARETAKER_PUSH_HOST_REGS
+ movq %rsp, CXP_STACK_ORIG(%rdi)
+
+ /* Switch to private preserved stack */
+ movq CXP_STACK_TOP(%rdi), %rsp
+
+ /* Activate VMCS on this pCPU */
+ vmptrld CXP_VMCS_PA(%rdi)
+
+ /* Restore guest MSR_KERNEL_GS_BASE if valid/canonical */
+ movq CXP_KERNEL_GS_BASE(%rdi), %rax
+ movq %rax, %rdx
+ sarq $56, %rdx
+ incq %rdx
+ cmpq $1, %rdx
+ ja 2f
+ movl $MSR_KERNEL_GS_BASE, %ecx
+ movq CXP_KERNEL_GS_BASE(%rdi), %rax
+ movq %rax, %rdx
+ shrq $32, %rdx
+1: wrmsr
+2:
+ _ASM_EXTABLE(1b, 2b)
+
+ /* Restore guest GPRs */
+ CARETAKER_RESTORE_GPRS %rdi
+
+ pushq %rdi
+ movq CXP_REG_RDI(%rdi), %rdi
+
+ /* Clear CPU microarchitectural buffers before guest entry */
+ CARETAKER_CLEAR_CPU_BUFFERS
+
+ /* Launch or resume */
+ vmresume
+ vmlaunch
+
+ /* If both fail, record error and return */
+ popq %rdi
+ mov $VMCS_VM_INSTRUCTION_ERROR, %eax
+ vmread %rax, %rax
+ movq %rax, CXP_LAST_EXIT_CODE(%rdi)
+ btsq $31, %rax
+ jmp .Lvmx_ret
+SYM_FUNC_END(vmx_caretaker_enter)
+
+SYM_FUNC_START(vmx_caretaker_exit_handler)
+ ENDBR
+ /*
+ * Hardware jumps here on VM-Exit with:
+ * RSP = HOST_RSP (CXP_STACK_TOP)
+ * CR3 = HOST_CR3
+ */
+ pushq %rdi
+
+ /* Recompute CVP pointer from RSP: CVP = (RSP & PAGE_MASK) - CXP_STACK_OFFSET */
+ movq %rsp, %rdi
+ andq $PAGE_MASK, %rdi
+ subq $CXP_STACK_OFFSET, %rdi
+
+ /* Save guest GPRs into CVP */
+ CARETAKER_SAVE_GPRS %rdi
+ popq CXP_REG_RDI(%rdi)
+
+ /* Stuff Return Stack Buffer to prevent guest RSB poisoning */
+ CARETAKER_FILL_RETURN_BUFFER %rax
+
+ /* Read guest MSR_KERNEL_GS_BASE */
+ movl $MSR_KERNEL_GS_BASE, %ecx
+ rdmsr
+ shlq $32, %rdx
+ orq %rax, %rdx
+ movq %rdx, CXP_KERNEL_GS_BASE(%rdi)
+
+ xorq %rax, %rax
+ jmp .Lvmx_ret
+
+.Lvmx_ret:
+ /* Restore caller stack */
+ movq CXP_STACK_ORIG(%rdi), %rsp
+
+ /* Restore host callee-saved registers */
+ CARETAKER_POP_HOST_REGS
+ ret
+SYM_FUNC_END(vmx_caretaker_exit_handler)
+STACK_FRAME_NON_STANDARD(vmx_caretaker_enter)
+STACK_FRAME_NON_STANDARD(vmx_caretaker_exit_handler)
--
2.55.0.1082.g2b9226bbc0-goog