[PATCH 14/17] KVM: nSVM: Use a gfn_to_pfn_cache for the vmcb12 page

[David Woodhouse]

From: David Woodhouse <dwmw@xxxxxxxxxxxx>

Replace the transient map/unmap of the vmcb12 page on every nested
VMRUN (control/save copy-in) and nested VM-exit (state copy-back) with
a gfn_to_pfn_cache keyed on vmcb12_gpa. With unmanaged guest memory
the transient map is a memremap/memunmap cycle on every L1<->L2
transition; the cache persists the kernel mapping for as long as the
gPA and its translation are unchanged.

No pinning is needed, unlike the nVMX APIC pages: SVM never hands an
L1-owned physical address to the CPU in vmcb02 (the merged MSRPM and
the IOPM are kernel-owned pages), so the vmcb12 page is only ever
accessed by KVM itself, under the SRCU read lock, with the usual
check/refresh protocol.

Dirty marking moves from unconditional-at-unmap to the actual write
paths: the SVM_EXIT_ERR reflection in the VMRUN consistency-check
failure path, and the copy-back at nested VM-exit.

The SMM enter/leave and VMLOAD/VMSAVE emulation paths keep the
transient map: they are cold, and VMLOAD/VMSAVE takes an arbitrary
gPA from RAX which would thrash a single-slot cache.

Signed-off-by: David Woodhouse <dwmw@xxxxxxxxxxxx>
Assisted-by: Claude:claude-mythos-5
---
 arch/x86/kvm/svm/nested.c | 26 +++++++++++++++++++-------
 arch/x86/kvm/svm/svm.h    |  3 +++
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index acc423b13445..a8ee0a03d9f7 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -1088,12 +1088,13 @@ static int nested_svm_copy_vmcb12_to_cache(struct kvm_vcpu *vcpu, u64 vmcb12_gpa
 	struct vcpu_svm *svm = to_svm(vcpu);
 	struct vmcb *vmcb12;
 	int r = 0;
+	int idx;
 
-	CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(vmcb12_gpa));
-	if (m.ret)
+	idx = kvm_gpc_lock_page(&svm->nested.vmcb12_cache, vmcb12_gpa);
+	if (idx < 0)
 		return -EFAULT;
 
-	vmcb12 = m.map.hva;
+	vmcb12 = svm->nested.vmcb12_cache.khva;
 	nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
 	nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
 
@@ -1104,9 +1105,11 @@ static int nested_svm_copy_vmcb12_to_cache(struct kvm_vcpu *vcpu, u64 vmcb12_gpa
 		vmcb12->control.event_inj = 0;
 		vmcb12->control.event_inj_err = 0;
 		svm_set_gif(svm, false);
+		kvm_gpc_mark_dirty_in_slot(&svm->nested.vmcb12_cache);
 		r = -EINVAL;
 	}
 
+	kvm_gpc_unlock(&svm->nested.vmcb12_cache, idx);
 	return r;
 }
 
@@ -1251,12 +1254,14 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu)
 	struct vcpu_svm *svm = to_svm(vcpu);
 	struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
 	struct vmcb *vmcb12;
+	int idx;
 
-	CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa));
-	if (m.ret)
-		return m.ret;
+	idx = kvm_gpc_lock_page(&svm->nested.vmcb12_cache,
+				  svm->nested.vmcb12_gpa);
+	if (idx < 0)
+		return idx;
 
-	vmcb12 = m.map.hva;
+	vmcb12 = svm->nested.vmcb12_cache.khva;
 
 	vmcb12->save.es     = vmcb02->save.es;
 	vmcb12->save.cs     = vmcb02->save.cs;
@@ -1311,6 +1316,9 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu)
 				       vmcb12->control.exit_int_info_err,
 				       KVM_ISA_SVM);
 
+	kvm_gpc_mark_dirty_in_slot(&svm->nested.vmcb12_cache);
+	kvm_gpc_unlock(&svm->nested.vmcb12_cache, idx);
+
 	return 0;
 }
 
@@ -1494,6 +1502,8 @@ int svm_allocate_nested(struct vcpu_svm *svm)
 	if (!svm->nested.msrpm)
 		goto err_free_vmcb02;
 
+	kvm_gpc_init(&svm->nested.vmcb12_cache, svm->vcpu.kvm);
+
 	svm->nested.initialized = true;
 	return 0;
 
@@ -1513,6 +1523,8 @@ void svm_free_nested(struct vcpu_svm *svm)
 	svm_vcpu_free_msrpm(svm->nested.msrpm);
 	svm->nested.msrpm = NULL;
 
+	kvm_gpc_deactivate(&svm->nested.vmcb12_cache);
+
 	__free_page(virt_to_page(svm->nested.vmcb02.ptr));
 	svm->nested.vmcb02.ptr = NULL;
 
diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index e958943b8162..cd0755919062 100644
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -218,6 +218,9 @@ struct svm_nested_state {
 	u64 last_vmcb12_gpa;
 	u64 last_bus_lock_rip;
 
+	/* Cached mapping of the vmcb12 page, keyed on vmcb12_gpa */
+	struct gfn_to_pfn_cache vmcb12_cache;
+
 	/*
 	 * The MSR permissions map used for vmcb02, which is the merge result
 	 * of vmcb01 and vmcb12
-- 
2.55.0